2024-11-13 05:50:10,344 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 05:50:10,357 main DEBUG Took 0.011035 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 05:50:10,358 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 05:50:10,358 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 05:50:10,359 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 05:50:10,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,369 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 05:50:10,383 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,385 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,386 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,386 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,387 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,389 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,389 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,390 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,391 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,392 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,392 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,393 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,394 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,394 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,395 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,395 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,396 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,396 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:50:10,397 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,397 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 05:50:10,399 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:50:10,400 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 05:50:10,403 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 05:50:10,404 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 05:50:10,405 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 05:50:10,406 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 05:50:10,415 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 05:50:10,417 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 05:50:10,419 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 05:50:10,419 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 05:50:10,420 main DEBUG createAppenders(={Console}) 2024-11-13 05:50:10,420 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 05:50:10,421 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 05:50:10,421 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 05:50:10,421 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 05:50:10,421 main DEBUG OutputStream closed 2024-11-13 05:50:10,422 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 05:50:10,422 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 05:50:10,422 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 05:50:10,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 05:50:10,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 05:50:10,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 05:50:10,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 05:50:10,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 05:50:10,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 05:50:10,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 05:50:10,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 05:50:10,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 05:50:10,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 05:50:10,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 05:50:10,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 05:50:10,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 05:50:10,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 05:50:10,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 05:50:10,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 05:50:10,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 05:50:10,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 05:50:10,503 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 05:50:10,503 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 05:50:10,503 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 05:50:10,504 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T05:50:10,748 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d 2024-11-13 05:50:10,752 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 05:50:10,752 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T05:50:10,760 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-13T05:50:10,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:50:10,809 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259, deleteOnExit=true 2024-11-13T05:50:10,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:50:10,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/test.cache.data in system properties and HBase conf 2024-11-13T05:50:10,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:50:10,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:50:10,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:50:10,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:50:10,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:50:10,908 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T05:50:11,012 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:50:11,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:50:11,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:50:11,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:50:11,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:50:11,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:50:11,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:50:11,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:50:11,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:50:11,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:50:11,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:50:11,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:50:11,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:50:11,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:50:11,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:50:12,119 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T05:50:12,199 INFO [Time-limited test {}] log.Log(170): Logging initialized @2523ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T05:50:12,266 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:50:12,327 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:50:12,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:50:12,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:50:12,354 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:50:12,371 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:50:12,375 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:50:12,376 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:50:12,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/java.io.tmpdir/jetty-localhost-41353-hadoop-hdfs-3_4_1-tests_jar-_-any-13577472479003361023/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:50:12,565 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:41353} 2024-11-13T05:50:12,566 INFO [Time-limited test {}] server.Server(415): Started @2891ms 2024-11-13T05:50:13,208 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:50:13,217 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:50:13,219 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:50:13,219 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:50:13,219 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:50:13,220 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:50:13,221 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:50:13,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/java.io.tmpdir/jetty-localhost-38615-hadoop-hdfs-3_4_1-tests_jar-_-any-6313394681406838413/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:50:13,343 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:38615} 2024-11-13T05:50:13,344 INFO [Time-limited test {}] server.Server(415): Started @3669ms 2024-11-13T05:50:13,417 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:50:14,334 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/data/data2/current/BP-959269583-172.17.0.2-1731477011648/current, will proceed with Du for space computation calculation, 2024-11-13T05:50:14,334 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/data/data1/current/BP-959269583-172.17.0.2-1731477011648/current, will proceed with Du for space computation calculation, 2024-11-13T05:50:14,380 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:50:14,443 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd14ab8b18addd98 with lease ID 0xcf4daecdb5053a71: Processing first storage report for DS-576893c1-124a-4fb3-aab7-4bfa7dc32875 from datanode DatanodeRegistration(127.0.0.1:34273, datanodeUuid=a6f45347-bf4f-4995-8b21-bcffbf3db50e, infoPort=38723, infoSecurePort=0, ipcPort=44067, storageInfo=lv=-57;cid=testClusterID;nsid=907769702;c=1731477011648) 2024-11-13T05:50:14,444 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd14ab8b18addd98 with lease ID 0xcf4daecdb5053a71: from storage DS-576893c1-124a-4fb3-aab7-4bfa7dc32875 node DatanodeRegistration(127.0.0.1:34273, datanodeUuid=a6f45347-bf4f-4995-8b21-bcffbf3db50e, infoPort=38723, infoSecurePort=0, ipcPort=44067, storageInfo=lv=-57;cid=testClusterID;nsid=907769702;c=1731477011648), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:50:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbd14ab8b18addd98 with lease ID 0xcf4daecdb5053a71: Processing first storage report for DS-2edfda92-5246-4485-a8f8-4e82a6f84b0c from datanode DatanodeRegistration(127.0.0.1:34273, datanodeUuid=a6f45347-bf4f-4995-8b21-bcffbf3db50e, infoPort=38723, infoSecurePort=0, ipcPort=44067, storageInfo=lv=-57;cid=testClusterID;nsid=907769702;c=1731477011648) 2024-11-13T05:50:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbd14ab8b18addd98 with lease ID 0xcf4daecdb5053a71: from storage DS-2edfda92-5246-4485-a8f8-4e82a6f84b0c node DatanodeRegistration(127.0.0.1:34273, datanodeUuid=a6f45347-bf4f-4995-8b21-bcffbf3db50e, infoPort=38723, infoSecurePort=0, ipcPort=44067, storageInfo=lv=-57;cid=testClusterID;nsid=907769702;c=1731477011648), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:50:14,498 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d 2024-11-13T05:50:14,578 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/zookeeper_0, clientPort=49495, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:50:14,589 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49495 2024-11-13T05:50:14,601 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:14,604 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:14,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:50:15,242 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 with version=8 2024-11-13T05:50:15,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/hbase-staging 2024-11-13T05:50:15,332 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T05:50:15,575 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:50:15,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:50:15,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:50:15,596 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:50:15,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:50:15,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:50:15,756 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:50:15,819 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T05:50:15,831 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T05:50:15,835 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:50:15,864 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 36847 (auto-detected) 2024-11-13T05:50:15,865 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T05:50:15,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44267 2024-11-13T05:50:15,907 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44267 connecting to ZooKeeper ensemble=127.0.0.1:49495 2024-11-13T05:50:16,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442670x0, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:50:16,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44267-0x10134081ae10000 connected 2024-11-13T05:50:16,118 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:16,121 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:16,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:50:16,134 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468, hbase.cluster.distributed=false 2024-11-13T05:50:16,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:50:16,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44267 2024-11-13T05:50:16,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44267 2024-11-13T05:50:16,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44267 2024-11-13T05:50:16,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44267 2024-11-13T05:50:16,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44267 2024-11-13T05:50:16,266 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:50:16,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:50:16,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:50:16,268 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:50:16,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:50:16,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:50:16,271 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:50:16,274 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:50:16,281 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38979 2024-11-13T05:50:16,283 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38979 connecting to ZooKeeper ensemble=127.0.0.1:49495 2024-11-13T05:50:16,285 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:16,287 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:16,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389790x0, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:50:16,301 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:389790x0, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:50:16,301 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38979-0x10134081ae10001 connected 2024-11-13T05:50:16,304 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:50:16,312 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:50:16,315 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:50:16,321 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:50:16,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38979 2024-11-13T05:50:16,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38979 2024-11-13T05:50:16,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38979 2024-11-13T05:50:16,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38979 2024-11-13T05:50:16,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38979 2024-11-13T05:50:16,342 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bc386b7bf2fe:44267 2024-11-13T05:50:16,343 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:16,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:50:16,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:50:16,361 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:16,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:16,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:50:16,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:16,393 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:50:16,394 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bc386b7bf2fe,44267,1731477015408 from backup master directory 2024-11-13T05:50:16,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:16,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:50:16,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:50:16,401 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:50:16,401 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:16,404 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T05:50:16,406 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T05:50:16,489 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/hbase.id] with ID: 0f8f3e79-255c-44e7-af82-b12e7afebc35 2024-11-13T05:50:16,489 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/.tmp/hbase.id 2024-11-13T05:50:16,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:50:16,902 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/.tmp/hbase.id]:[hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/hbase.id] 2024-11-13T05:50:16,954 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:50:16,959 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:50:16,978 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-13T05:50:17,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:17,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:17,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:50:17,440 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:50:17,443 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:50:17,461 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:17,465 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:50:17,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:50:17,510 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store 2024-11-13T05:50:17,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:50:17,941 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T05:50:17,945 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:17,946 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:50:17,947 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:50:17,947 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:50:17,948 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:50:17,948 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:50:17,948 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:50:17,949 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477017946Disabling compacts and flushes for region at 1731477017946Disabling writes for close at 1731477017948 (+2 ms)Writing region close event to WAL at 1731477017948Closed at 1731477017948 2024-11-13T05:50:17,952 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/.initializing 2024-11-13T05:50:17,952 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/WALs/bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:17,960 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:50:17,975 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C44267%2C1731477015408, suffix=, logDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/WALs/bc386b7bf2fe,44267,1731477015408, archiveDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/oldWALs, maxLogs=10 2024-11-13T05:50:17,996 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/WALs/bc386b7bf2fe,44267,1731477015408/bc386b7bf2fe%2C44267%2C1731477015408.1731477017979, exclude list is [], retry=0 2024-11-13T05:50:18,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34273,DS-576893c1-124a-4fb3-aab7-4bfa7dc32875,DISK] 2024-11-13T05:50:18,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T05:50:18,043 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/WALs/bc386b7bf2fe,44267,1731477015408/bc386b7bf2fe%2C44267%2C1731477015408.1731477017979 2024-11-13T05:50:18,044 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38723:38723)] 2024-11-13T05:50:18,045 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:50:18,045 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:18,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,049 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:50:18,109 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:18,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:18,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:50:18,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:18,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:18,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:50:18,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:18,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:18,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:50:18,128 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:18,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:18,130 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,134 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,135 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,141 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,142 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,145 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:50:18,149 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:50:18,154 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:50:18,156 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73945912, jitterRate=0.10187995433807373}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:50:18,161 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731477018060Initializing all the Stores at 1731477018062 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477018063 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477018063Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477018063Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477018064 (+1 ms)Cleaning up temporary data from old regions at 1731477018142 (+78 ms)Region opened successfully at 1731477018161 (+19 ms) 2024-11-13T05:50:18,163 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:50:18,454 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2c71b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:50:18,480 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:50:18,490 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:50:18,490 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:50:18,493 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:50:18,494 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:50:18,499 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T05:50:18,499 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:50:18,523 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:50:18,532 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:50:18,649 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:50:18,652 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:50:18,653 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:50:18,691 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:50:18,693 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:50:18,697 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:50:18,733 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:50:18,734 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:50:18,775 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:50:18,791 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:50:18,841 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:50:18,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:50:18,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:50:18,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:18,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:18,880 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bc386b7bf2fe,44267,1731477015408, sessionid=0x10134081ae10000, setting cluster-up flag (Was=false) 2024-11-13T05:50:18,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:18,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:19,108 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:50:19,110 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:19,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:19,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:19,399 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:50:19,401 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:19,406 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:50:19,431 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(746): ClusterId : 0f8f3e79-255c-44e7-af82-b12e7afebc35 2024-11-13T05:50:19,433 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:50:19,477 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:50:19,478 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:50:19,477 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:50:19,486 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:50:19,492 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:50:19,497 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bc386b7bf2fe,44267,1731477015408 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:50:19,526 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:50:19,527 DEBUG [RS:0;bc386b7bf2fe:38979 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560dc1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:50:19,527 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:50:19,528 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:50:19,528 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:50:19,528 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:50:19,528 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bc386b7bf2fe:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:50:19,528 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,528 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:50:19,529 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,533 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477049533 2024-11-13T05:50:19,534 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:50:19,535 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:50:19,535 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:50:19,536 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:50:19,541 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:50:19,541 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:19,542 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:50:19,542 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:50:19,542 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:50:19,542 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:50:19,543 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bc386b7bf2fe:38979 2024-11-13T05:50:19,543 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,547 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:50:19,547 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:50:19,547 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:50:19,547 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:50:19,548 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:50:19,549 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:50:19,550 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,44267,1731477015408 with port=38979, startcode=1731477016232 2024-11-13T05:50:19,552 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:50:19,552 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:50:19,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:50:19,554 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477019553,5,FailOnTimeoutGroup] 2024-11-13T05:50:19,554 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477019554,5,FailOnTimeoutGroup] 2024-11-13T05:50:19,554 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,554 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:50:19,556 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,556 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,563 DEBUG [RS:0;bc386b7bf2fe:38979 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:50:19,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:50:19,605 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44267 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:19,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44267 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:19,624 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:50:19,624 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41183 2024-11-13T05:50:19,624 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:50:19,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:50:19,662 DEBUG [RS:0;bc386b7bf2fe:38979 {}] zookeeper.ZKUtil(111): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:19,662 WARN [RS:0;bc386b7bf2fe:38979 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:50:19,662 INFO [RS:0;bc386b7bf2fe:38979 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:50:19,663 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:19,665 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,38979,1731477016232] 2024-11-13T05:50:19,690 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:50:19,708 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:50:19,714 INFO [RS:0;bc386b7bf2fe:38979 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:50:19,715 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,716 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:50:19,723 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:50:19,724 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:50:19,725 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,726 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,726 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,726 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,726 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,726 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:50:19,726 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:50:19,727 DEBUG [RS:0;bc386b7bf2fe:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:50:19,727 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,728 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,728 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,728 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,728 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,728 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38979,1731477016232-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:50:19,747 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:50:19,748 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38979,1731477016232-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,748 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,749 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.Replication(171): bc386b7bf2fe,38979,1731477016232 started 2024-11-13T05:50:19,766 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:19,766 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,38979,1731477016232, RpcServer on bc386b7bf2fe/172.17.0.2:38979, sessionid=0x10134081ae10001 2024-11-13T05:50:19,767 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:50:19,767 DEBUG [RS:0;bc386b7bf2fe:38979 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:19,767 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,38979,1731477016232' 2024-11-13T05:50:19,768 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:50:19,769 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:50:19,769 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:50:19,769 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:50:19,769 DEBUG [RS:0;bc386b7bf2fe:38979 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:19,770 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,38979,1731477016232' 2024-11-13T05:50:19,770 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:50:19,770 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:50:19,771 DEBUG [RS:0;bc386b7bf2fe:38979 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:50:19,771 INFO [RS:0;bc386b7bf2fe:38979 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:50:19,771 INFO [RS:0;bc386b7bf2fe:38979 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:50:19,876 INFO [RS:0;bc386b7bf2fe:38979 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:50:19,879 INFO [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C38979%2C1731477016232, suffix=, logDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232, archiveDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/oldWALs, maxLogs=32 2024-11-13T05:50:19,895 DEBUG [RS:0;bc386b7bf2fe:38979 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232/bc386b7bf2fe%2C38979%2C1731477016232.1731477019881, exclude list is [], retry=0 2024-11-13T05:50:19,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34273,DS-576893c1-124a-4fb3-aab7-4bfa7dc32875,DISK] 2024-11-13T05:50:19,902 INFO [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232/bc386b7bf2fe%2C38979%2C1731477016232.1731477019881 2024-11-13T05:50:19,903 DEBUG [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38723:38723)] 2024-11-13T05:50:19,955 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:50:19,956 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:50:19,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741833_1009 (size=32) 2024-11-13T05:50:20,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:20,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:50:20,375 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:50:20,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:50:20,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:50:20,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:50:20,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:50:20,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:50:20,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:50:20,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:50:20,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740 2024-11-13T05:50:20,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740 2024-11-13T05:50:20,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:50:20,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:50:20,395 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:50:20,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:50:20,402 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:50:20,403 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59957889, jitterRate=-0.10655783116817474}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:50:20,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731477020370Initializing all the Stores at 1731477020372 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477020372Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477020372Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477020372Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477020372Cleaning up temporary data from old regions at 1731477020394 (+22 ms)Region opened successfully at 1731477020406 (+12 ms) 2024-11-13T05:50:20,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:50:20,406 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:50:20,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:50:20,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:50:20,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:50:20,408 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:50:20,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477020406Disabling compacts and flushes for region at 1731477020406Disabling writes for close at 1731477020406Writing region close event to WAL at 1731477020407 (+1 ms)Closed at 1731477020408 (+1 ms) 2024-11-13T05:50:20,411 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:50:20,411 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:50:20,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:50:20,425 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:50:20,427 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:50:20,580 DEBUG [bc386b7bf2fe:44267 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:50:20,590 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:20,596 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38979,1731477016232, state=OPENING 2024-11-13T05:50:20,624 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:50:20,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:20,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:50:20,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:50:20,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:50:20,635 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:50:20,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:50:20,809 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:50:20,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60263, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:50:20,826 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:50:20,826 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:50:20,827 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:50:20,831 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C38979%2C1731477016232.meta, suffix=.meta, logDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232, archiveDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/oldWALs, maxLogs=32 2024-11-13T05:50:20,845 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232/bc386b7bf2fe%2C38979%2C1731477016232.meta.1731477020832.meta, exclude list is [], retry=0 2024-11-13T05:50:20,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34273,DS-576893c1-124a-4fb3-aab7-4bfa7dc32875,DISK] 2024-11-13T05:50:20,853 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/WALs/bc386b7bf2fe,38979,1731477016232/bc386b7bf2fe%2C38979%2C1731477016232.meta.1731477020832.meta 2024-11-13T05:50:20,854 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38723:38723)] 2024-11-13T05:50:20,854 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:50:20,856 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:50:20,858 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:50:20,862 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:50:20,866 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:50:20,867 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:20,867 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:50:20,867 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:50:20,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:50:20,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:50:20,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:50:20,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:50:20,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:50:20,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:50:20,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:50:20,880 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:50:20,880 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:20,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:50:20,881 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:50:20,882 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740 2024-11-13T05:50:20,885 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740 2024-11-13T05:50:20,887 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:50:20,887 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:50:20,888 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:50:20,890 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:50:20,892 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66776252, jitterRate=-0.004956305027008057}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:50:20,892 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:50:20,894 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731477020868Writing region info on filesystem at 1731477020868Initializing all the Stores at 1731477020870 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477020870Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477020870Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477020870Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477020870Cleaning up temporary data from old regions at 1731477020887 (+17 ms)Running coprocessor post-open hooks at 1731477020892 (+5 ms)Region opened successfully at 1731477020893 (+1 ms) 2024-11-13T05:50:20,900 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731477020801 2024-11-13T05:50:20,909 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:50:20,910 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:50:20,912 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:20,914 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38979,1731477016232, state=OPEN 2024-11-13T05:50:20,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:50:20,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:50:20,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:50:20,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:50:20,983 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:20,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:50:20,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,38979,1731477016232 in 347 msec 2024-11-13T05:50:20,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:50:20,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 573 msec 2024-11-13T05:50:20,997 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:50:20,997 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:50:21,020 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:21,022 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:21,040 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:21,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:21,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6280 sec 2024-11-13T05:50:21,064 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731477021064, completionTime=-1 2024-11-13T05:50:21,067 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T05:50:21,067 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:50:21,092 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-13T05:50:21,092 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477081092 2024-11-13T05:50:21,092 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477141092 2024-11-13T05:50:21,092 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-13T05:50:21,095 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,44267,1731477015408-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,095 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,44267,1731477015408-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,095 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,44267,1731477015408-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,096 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bc386b7bf2fe:44267, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,097 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,097 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,105 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:50:21,131 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.730sec 2024-11-13T05:50:21,132 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:50:21,133 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:50:21,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:50:21,135 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:50:21,135 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:50:21,136 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,44267,1731477015408-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:50:21,137 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,44267,1731477015408-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:50:21,145 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:50:21,146 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:50:21,146 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,44267,1731477015408-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:50:21,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:21,242 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T05:50:21,242 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T05:50:21,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:21,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:21,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:21,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:21,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775efbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:21,264 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:21,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:21,268 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:21,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:21,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:21,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=7132 2024-11-13T05:50:21,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:50:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:21,342 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1fd879ff 2024-11-13T05:50:21,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:50:21,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:50:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:50:21,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:50:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:21,368 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:50:21,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T05:50:21,370 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:21,373 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:50:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:50:21,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741835_1011 (size=1061) 2024-11-13T05:50:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:50:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:50:21,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:50:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741836_1012 (size=53) 2024-11-13T05:50:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:50:22,216 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:22,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9480a359b149cb565364fd0f1f82db91, disabling compactions & flushes 2024-11-13T05:50:22,217 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. after waiting 0 ms 2024-11-13T05:50:22,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,217 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,217 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9480a359b149cb565364fd0f1f82db91: Waiting for close lock at 1731477022216Disabling compacts and flushes for region at 1731477022216Disabling writes for close at 1731477022217 (+1 ms)Writing region close event to WAL at 1731477022217Closed at 1731477022217 2024-11-13T05:50:22,220 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:50:22,225 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731477022220"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477022220"}]},"ts":"1731477022220"} 2024-11-13T05:50:22,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:50:22,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:50:22,235 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477022232"}]},"ts":"1731477022232"} 2024-11-13T05:50:22,240 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:50:22,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, ASSIGN}] 2024-11-13T05:50:22,245 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, ASSIGN 2024-11-13T05:50:22,247 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=false 2024-11-13T05:50:22,398 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9480a359b149cb565364fd0f1f82db91, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:22,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, ASSIGN because future has completed 2024-11-13T05:50:22,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:50:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:50:22,564 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,564 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:50:22,565 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,565 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:22,565 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,565 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,568 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,571 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:22,572 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9480a359b149cb565364fd0f1f82db91 columnFamilyName A 2024-11-13T05:50:22,572 DEBUG [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:22,573 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.HStore(327): Store=9480a359b149cb565364fd0f1f82db91/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:22,573 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,576 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:22,576 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9480a359b149cb565364fd0f1f82db91 columnFamilyName B 2024-11-13T05:50:22,577 DEBUG [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:22,578 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.HStore(327): Store=9480a359b149cb565364fd0f1f82db91/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:22,578 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,580 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:22,581 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9480a359b149cb565364fd0f1f82db91 columnFamilyName C 2024-11-13T05:50:22,581 DEBUG [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:22,582 INFO [StoreOpener-9480a359b149cb565364fd0f1f82db91-1 {}] regionserver.HStore(327): Store=9480a359b149cb565364fd0f1f82db91/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:22,583 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,583 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,585 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,586 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,588 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,588 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,590 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:50:22,592 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,597 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:50:22,598 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9480a359b149cb565364fd0f1f82db91; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69918286, jitterRate=0.04186365008354187}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:50:22,598 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:22,599 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9480a359b149cb565364fd0f1f82db91: Running coprocessor pre-open hook at 1731477022565Writing region info on filesystem at 1731477022565Initializing all the Stores at 1731477022567 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477022567Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477022568 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477022568Cleaning up temporary data from old regions at 1731477022589 (+21 ms)Running coprocessor post-open hooks at 1731477022598 (+9 ms)Region opened successfully at 1731477022599 (+1 ms) 2024-11-13T05:50:22,602 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., pid=6, masterSystemTime=1731477022557 2024-11-13T05:50:22,606 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,606 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:22,607 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9480a359b149cb565364fd0f1f82db91, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:22,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:50:22,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:50:22,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 in 212 msec 2024-11-13T05:50:22,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:50:22,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, ASSIGN in 378 msec 2024-11-13T05:50:22,627 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:50:22,628 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477022627"}]},"ts":"1731477022627"} 2024-11-13T05:50:22,632 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:50:22,634 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:50:22,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2780 sec 2024-11-13T05:50:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:50:23,543 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:23,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4736cbb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29446fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e077d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,562 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:50:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:23,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d4e458b 2024-11-13T05:50:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:50:23,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:50:23,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,598 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,612 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,619 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,633 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd942f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dac934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2335953a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4331d0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563d13e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7163a69f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,669 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd14d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265884f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,676 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44731cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@870009d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7527002c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d95629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,692 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421214a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:23,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:23,696 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373a2bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:23,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33abc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:23,704 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,709 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,711 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:23,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,714 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,723 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T05:50:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:23,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:23,732 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,733 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,735 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:23,736 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,739 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:23,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:23,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:50:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:23,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:50:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:23,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:23,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:23,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:23,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:23,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:23,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:23,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:23,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:23,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:23,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477023985, max timestamp=1731477024006Num uniques 859; 2024-11-13T05:50:24,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477023985, max timestamp=1731477024006Num uniques 859; 2024-11-13T05:50:24,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:50:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:24,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:50:24,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477023985, max timestamp=1731477024006Num uniques 859; 2024-11-13T05:50:24,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:50:24,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/db91e3d831bf4eaf83bda4e137197948 is 50, key is test_row_0/A:col10/1731477023771/Put/seqid=0 2024-11-13T05:50:24,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741837_1013 (size=12001) 2024-11-13T05:50:24,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/db91e3d831bf4eaf83bda4e137197948 2024-11-13T05:50:24,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:24,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477084179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477084171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477084184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477084189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477084197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/de241b95421546b88ee089ed81cf39cb is 50, key is test_row_0/B:col10/1731477023771/Put/seqid=0 2024-11-13T05:50:24,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:24,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:24,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:24,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741838_1014 (size=12001) 2024-11-13T05:50:24,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/de241b95421546b88ee089ed81cf39cb 2024-11-13T05:50:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477084373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477084374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477084374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477084375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477084374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fbbcf7a3f3674961b3e7cf6e765f08c8 is 50, key is test_row_0/C:col10/1731477023771/Put/seqid=0 2024-11-13T05:50:24,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741839_1015 (size=12001) 2024-11-13T05:50:24,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fbbcf7a3f3674961b3e7cf6e765f08c8 2024-11-13T05:50:24,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/db91e3d831bf4eaf83bda4e137197948 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/db91e3d831bf4eaf83bda4e137197948 2024-11-13T05:50:24,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/db91e3d831bf4eaf83bda4e137197948, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T05:50:24,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/de241b95421546b88ee089ed81cf39cb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/de241b95421546b88ee089ed81cf39cb 2024-11-13T05:50:24,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/de241b95421546b88ee089ed81cf39cb, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T05:50:24,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fbbcf7a3f3674961b3e7cf6e765f08c8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fbbcf7a3f3674961b3e7cf6e765f08c8 2024-11-13T05:50:24,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fbbcf7a3f3674961b3e7cf6e765f08c8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T05:50:24,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 9480a359b149cb565364fd0f1f82db91 in 789ms, sequenceid=13, compaction requested=false 2024-11-13T05:50:24,631 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T05:50:24,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:24,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T05:50:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:24,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/b9b60a6a5e5c4374ac26e6d9c20210f7 is 50, key is test_row_0/A:col10/1731477024640/Put/seqid=0 2024-11-13T05:50:24,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477084676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741840_1016 (size=14341) 2024-11-13T05:50:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/b9b60a6a5e5c4374ac26e6d9c20210f7 2024-11-13T05:50:24,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477084689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477084693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477084714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477084716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1a01420dc631470b9926d92ee6c3a26d is 50, key is test_row_0/B:col10/1731477024640/Put/seqid=0 2024-11-13T05:50:24,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741841_1017 (size=12001) 2024-11-13T05:50:24,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:24,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:24,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477084844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:24,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477084872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477084874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477084874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477084874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477085085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477085103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477085104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477085105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477085106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1a01420dc631470b9926d92ee6c3a26d 2024-11-13T05:50:25,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6da2853fa141450783a190d8210fdffe is 50, key is test_row_0/C:col10/1731477024640/Put/seqid=0 2024-11-13T05:50:25,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741842_1018 (size=12001) 2024-11-13T05:50:25,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477085402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477085444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477085453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477085454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477085454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:25,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:25,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:25,683 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:50:25,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6da2853fa141450783a190d8210fdffe 2024-11-13T05:50:25,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/b9b60a6a5e5c4374ac26e6d9c20210f7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9b60a6a5e5c4374ac26e6d9c20210f7 2024-11-13T05:50:25,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9b60a6a5e5c4374ac26e6d9c20210f7, entries=200, sequenceid=48, filesize=14.0 K 2024-11-13T05:50:25,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1a01420dc631470b9926d92ee6c3a26d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1a01420dc631470b9926d92ee6c3a26d 2024-11-13T05:50:25,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1a01420dc631470b9926d92ee6c3a26d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T05:50:25,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6da2853fa141450783a190d8210fdffe as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6da2853fa141450783a190d8210fdffe 2024-11-13T05:50:25,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6da2853fa141450783a190d8210fdffe, entries=150, sequenceid=48, filesize=11.7 K 2024-11-13T05:50:25,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 9480a359b149cb565364fd0f1f82db91 in 1158ms, sequenceid=48, compaction requested=false 2024-11-13T05:50:25,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:25,815 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:50:25,815 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:50:25,823 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:50:25,823 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:50:25,824 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:50:25,824 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:50:25,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T05:50:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:25,833 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T05:50:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:25,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:25,840 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T05:50:25,840 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T05:50:25,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/e7332592e9ba4bd9b26edce30db2435c is 50, key is test_row_0/A:col10/1731477024653/Put/seqid=0 2024-11-13T05:50:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:25,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741843_1019 (size=12001) 2024-11-13T05:50:25,908 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/e7332592e9ba4bd9b26edce30db2435c 2024-11-13T05:50:25,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/d8050035cdbb47eab2026905ab489594 is 50, key is test_row_0/B:col10/1731477024653/Put/seqid=0 2024-11-13T05:50:25,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741844_1020 (size=12001) 2024-11-13T05:50:25,994 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/d8050035cdbb47eab2026905ab489594 2024-11-13T05:50:26,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5c365f00df464bd0b9a00c9b8f120bfe is 50, key is test_row_0/C:col10/1731477024653/Put/seqid=0 2024-11-13T05:50:26,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741845_1021 (size=12001) 2024-11-13T05:50:26,074 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5c365f00df464bd0b9a00c9b8f120bfe 2024-11-13T05:50:26,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477026043, max timestamp=1731477026059Num uniques 859; 2024-11-13T05:50:26,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:26,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477026043, max timestamp=1731477026059Num uniques 859; 2024-11-13T05:50:26,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/e7332592e9ba4bd9b26edce30db2435c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e7332592e9ba4bd9b26edce30db2435c 2024-11-13T05:50:26,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:26,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731477026043, max timestamp=1731477026059Num uniques 859; 2024-11-13T05:50:26,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-13T05:50:26,111 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e7332592e9ba4bd9b26edce30db2435c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T05:50:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/d8050035cdbb47eab2026905ab489594 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d8050035cdbb47eab2026905ab489594 2024-11-13T05:50:26,135 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d8050035cdbb47eab2026905ab489594, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T05:50:26,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5c365f00df464bd0b9a00c9b8f120bfe as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5c365f00df464bd0b9a00c9b8f120bfe 2024-11-13T05:50:26,171 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5c365f00df464bd0b9a00c9b8f120bfe, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T05:50:26,173 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=106.95 KB/109515 for 9480a359b149cb565364fd0f1f82db91 in 340ms, sequenceid=59, compaction requested=true 2024-11-13T05:50:26,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:26,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:26,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T05:50:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T05:50:26,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=113.66 KB heapSize=324.30 KB 2024-11-13T05:50:26,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:26,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:26,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:26,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:26,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:26,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:26,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:50:26,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4390 sec 2024-11-13T05:50:26,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.4670 sec 2024-11-13T05:50:26,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9982fca6c3214783a20bc35eb0c4be21 is 50, key is test_row_0/A:col10/1731477026164/Put/seqid=0 2024-11-13T05:50:26,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741846_1022 (size=19021) 2024-11-13T05:50:26,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9982fca6c3214783a20bc35eb0c4be21 2024-11-13T05:50:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477086223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477086242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477086243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477086253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/c8828ed585f44a9c81b07b703b609733 is 50, key is test_row_0/B:col10/1731477026164/Put/seqid=0 2024-11-13T05:50:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477086253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741847_1023 (size=12001) 2024-11-13T05:50:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477086363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477086382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477086382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477086385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,401 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:50:26,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477086402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477086571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477086603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477086603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477086604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477086613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/c8828ed585f44a9c81b07b703b609733 2024-11-13T05:50:26,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1891309586394b7797529476cc441def is 50, key is test_row_0/C:col10/1731477026164/Put/seqid=0 2024-11-13T05:50:26,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741848_1024 (size=12001) 2024-11-13T05:50:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477086882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477086922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477086922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477086922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:26,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477086932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1891309586394b7797529476cc441def 2024-11-13T05:50:27,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9982fca6c3214783a20bc35eb0c4be21 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9982fca6c3214783a20bc35eb0c4be21 2024-11-13T05:50:27,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9982fca6c3214783a20bc35eb0c4be21, entries=300, sequenceid=93, filesize=18.6 K 2024-11-13T05:50:27,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/c8828ed585f44a9c81b07b703b609733 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/c8828ed585f44a9c81b07b703b609733 2024-11-13T05:50:27,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/c8828ed585f44a9c81b07b703b609733, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T05:50:27,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1891309586394b7797529476cc441def as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1891309586394b7797529476cc441def 2024-11-13T05:50:27,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1891309586394b7797529476cc441def, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T05:50:27,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~113.66 KB/116385, heapSize ~324.26 KB/332040, currentSize=67.09 KB/68700 for 9480a359b149cb565364fd0f1f82db91 in 1106ms, sequenceid=93, compaction requested=true 2024-11-13T05:50:27,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:27,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:27,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:27,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:27,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:27,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:27,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:27,307 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:27,311 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:27,313 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:27,314 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:27,315 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:27,315 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:27,316 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:27,316 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:27,316 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/db91e3d831bf4eaf83bda4e137197948, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9b60a6a5e5c4374ac26e6d9c20210f7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e7332592e9ba4bd9b26edce30db2435c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9982fca6c3214783a20bc35eb0c4be21] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=56.0 K 2024-11-13T05:50:27,316 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/de241b95421546b88ee089ed81cf39cb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1a01420dc631470b9926d92ee6c3a26d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d8050035cdbb47eab2026905ab489594, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/c8828ed585f44a9c81b07b703b609733] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=46.9 K 2024-11-13T05:50:27,319 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting de241b95421546b88ee089ed81cf39cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731477023771 2024-11-13T05:50:27,319 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting db91e3d831bf4eaf83bda4e137197948, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731477023771 2024-11-13T05:50:27,320 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b60a6a5e5c4374ac26e6d9c20210f7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731477024178 2024-11-13T05:50:27,320 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a01420dc631470b9926d92ee6c3a26d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731477024182 2024-11-13T05:50:27,321 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7332592e9ba4bd9b26edce30db2435c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477024653 2024-11-13T05:50:27,322 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d8050035cdbb47eab2026905ab489594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477024653 2024-11-13T05:50:27,322 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9982fca6c3214783a20bc35eb0c4be21, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731477026120 2024-11-13T05:50:27,323 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting c8828ed585f44a9c81b07b703b609733, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731477026120 2024-11-13T05:50:27,411 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#13 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:27,412 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/d9ae103b09ec42179089bc1b4a3962ae is 50, key is test_row_0/A:col10/1731477026164/Put/seqid=0 2024-11-13T05:50:27,417 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#12 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:27,418 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/277ed23bd572469690923c228eebe18d is 50, key is test_row_0/B:col10/1731477026164/Put/seqid=0 2024-11-13T05:50:27,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:50:27,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:27,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:27,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:27,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:27,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:27,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:27,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/735fcc2a96094f938973992eea520ff3 is 50, key is test_row_0/A:col10/1731477026240/Put/seqid=0 2024-11-13T05:50:27,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741850_1026 (size=12139) 2024-11-13T05:50:27,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741849_1025 (size=12139) 2024-11-13T05:50:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741851_1027 (size=14341) 2024-11-13T05:50:27,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477087615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477087636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477087642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477087656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477087658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477087762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477087762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477087762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477087772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477087792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T05:50:27,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:27,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T05:50:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:27,930 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:27,933 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:27,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:27,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/735fcc2a96094f938973992eea520ff3 2024-11-13T05:50:27,950 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/277ed23bd572469690923c228eebe18d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/277ed23bd572469690923c228eebe18d 2024-11-13T05:50:27,951 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/d9ae103b09ec42179089bc1b4a3962ae as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d9ae103b09ec42179089bc1b4a3962ae 2024-11-13T05:50:27,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1c504ae1e6184d7aa51f7c7c1152945f is 50, key is test_row_0/B:col10/1731477026240/Put/seqid=0 2024-11-13T05:50:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477087982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477087982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477087982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:27,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477087997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,006 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 277ed23bd572469690923c228eebe18d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:28,006 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:28,009 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into d9ae103b09ec42179089bc1b4a3962ae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:28,009 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:28,010 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=12, startTime=1731477027285; duration=0sec 2024-11-13T05:50:28,010 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=12, startTime=1731477027304; duration=0sec 2024-11-13T05:50:28,011 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:28,011 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:28,011 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:28,012 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:28,012 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:28,015 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:28,015 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:28,015 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,016 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fbbcf7a3f3674961b3e7cf6e765f08c8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6da2853fa141450783a190d8210fdffe, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5c365f00df464bd0b9a00c9b8f120bfe, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1891309586394b7797529476cc441def] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=46.9 K 2024-11-13T05:50:28,017 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbcf7a3f3674961b3e7cf6e765f08c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731477023771 2024-11-13T05:50:28,018 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da2853fa141450783a190d8210fdffe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731477024182 2024-11-13T05:50:28,019 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c365f00df464bd0b9a00c9b8f120bfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477024653 2024-11-13T05:50:28,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741852_1028 (size=12001) 2024-11-13T05:50:28,021 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1891309586394b7797529476cc441def, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731477026120 2024-11-13T05:50:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477088012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:28,064 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:28,066 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5ca7980182de4c459eb7abbd55c0cff1 is 50, key is test_row_0/C:col10/1731477026164/Put/seqid=0 2024-11-13T05:50:28,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:28,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:28,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741853_1029 (size=12139) 2024-11-13T05:50:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:28,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:28,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477088302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477088302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477088312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477088312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477088332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:28,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:28,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1c504ae1e6184d7aa51f7c7c1152945f 2024-11-13T05:50:28,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/7b1f0fd615224c6b8a91e7a45efeaa17 is 50, key is test_row_0/C:col10/1731477026240/Put/seqid=0 2024-11-13T05:50:28,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741854_1030 (size=12001) 2024-11-13T05:50:28,535 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5ca7980182de4c459eb7abbd55c0cff1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5ca7980182de4c459eb7abbd55c0cff1 2024-11-13T05:50:28,551 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into 5ca7980182de4c459eb7abbd55c0cff1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:28,551 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:28,551 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=12, startTime=1731477027304; duration=0sec 2024-11-13T05:50:28,552 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:28,552 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:28,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:28,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:28,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477088822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477088822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477088822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477088822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477088844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:28,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:28,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:28,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/7b1f0fd615224c6b8a91e7a45efeaa17 2024-11-13T05:50:28,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/735fcc2a96094f938973992eea520ff3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/735fcc2a96094f938973992eea520ff3 2024-11-13T05:50:28,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/735fcc2a96094f938973992eea520ff3, entries=200, sequenceid=107, filesize=14.0 K 2024-11-13T05:50:28,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1c504ae1e6184d7aa51f7c7c1152945f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1c504ae1e6184d7aa51f7c7c1152945f 2024-11-13T05:50:28,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1c504ae1e6184d7aa51f7c7c1152945f, entries=150, sequenceid=107, filesize=11.7 K 2024-11-13T05:50:28,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/7b1f0fd615224c6b8a91e7a45efeaa17 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/7b1f0fd615224c6b8a91e7a45efeaa17 2024-11-13T05:50:29,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/7b1f0fd615224c6b8a91e7a45efeaa17, entries=150, sequenceid=107, filesize=11.7 K 2024-11-13T05:50:29,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 9480a359b149cb565364fd0f1f82db91 in 1585ms, sequenceid=107, compaction requested=false 2024-11-13T05:50:29,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:29,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T05:50:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:29,049 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T05:50:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:29,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:29,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/e631cabbd84a495b930f500baba07da1 is 50, key is test_row_0/A:col10/1731477027600/Put/seqid=0 2024-11-13T05:50:29,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741855_1031 (size=12001) 2024-11-13T05:50:29,537 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/e631cabbd84a495b930f500baba07da1 2024-11-13T05:50:29,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/829eb1a2551544058e98bd4fb3f17cf6 is 50, key is test_row_0/B:col10/1731477027600/Put/seqid=0 2024-11-13T05:50:29,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741856_1032 (size=12001) 2024-11-13T05:50:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:29,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477089896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:29,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477089897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:29,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:29,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:29,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477089901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:29,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477089906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477089899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477090012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477090012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477090016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,024 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/829eb1a2551544058e98bd4fb3f17cf6 2024-11-13T05:50:30,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477090034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477090034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/086166cb28984d5789328122b700b223 is 50, key is test_row_0/C:col10/1731477027600/Put/seqid=0 2024-11-13T05:50:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:30,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741857_1033 (size=12001) 2024-11-13T05:50:30,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477090222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477090232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477090232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477090252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477090259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,502 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/086166cb28984d5789328122b700b223 2024-11-13T05:50:30,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/e631cabbd84a495b930f500baba07da1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e631cabbd84a495b930f500baba07da1 2024-11-13T05:50:30,534 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e631cabbd84a495b930f500baba07da1, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T05:50:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/829eb1a2551544058e98bd4fb3f17cf6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/829eb1a2551544058e98bd4fb3f17cf6 2024-11-13T05:50:30,548 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/829eb1a2551544058e98bd4fb3f17cf6, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T05:50:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/086166cb28984d5789328122b700b223 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/086166cb28984d5789328122b700b223 2024-11-13T05:50:30,560 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/086166cb28984d5789328122b700b223, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T05:50:30,564 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9480a359b149cb565364fd0f1f82db91 in 1513ms, sequenceid=129, compaction requested=true 2024-11-13T05:50:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T05:50:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T05:50:30,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:50:30,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:30,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:30,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:30,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:30,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:30,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:30,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T05:50:30,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-13T05:50:30,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.6600 sec 2024-11-13T05:50:30,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/0e84963a6df24bdb9e22cc2d092b2a69 is 50, key is test_row_0/A:col10/1731477030568/Put/seqid=0 2024-11-13T05:50:30,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741858_1034 (size=12151) 2024-11-13T05:50:30,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477090618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477090622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477090629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477090636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477090636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477090741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477090743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477090752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477090752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477090753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477090951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477090964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477090964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477090964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:30,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477090964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/0e84963a6df24bdb9e22cc2d092b2a69 2024-11-13T05:50:31,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/30e602c490624a25a642024ba0345e4f is 50, key is test_row_0/B:col10/1731477030568/Put/seqid=0 2024-11-13T05:50:31,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741859_1035 (size=12151) 2024-11-13T05:50:31,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/30e602c490624a25a642024ba0345e4f 2024-11-13T05:50:31,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d6ecd69dbfe14439b0666324f69d9576 is 50, key is test_row_0/C:col10/1731477030568/Put/seqid=0 2024-11-13T05:50:31,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741860_1036 (size=12151) 2024-11-13T05:50:31,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d6ecd69dbfe14439b0666324f69d9576 2024-11-13T05:50:31,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/0e84963a6df24bdb9e22cc2d092b2a69 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0e84963a6df24bdb9e22cc2d092b2a69 2024-11-13T05:50:31,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0e84963a6df24bdb9e22cc2d092b2a69, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T05:50:31,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/30e602c490624a25a642024ba0345e4f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/30e602c490624a25a642024ba0345e4f 2024-11-13T05:50:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477091263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477091272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477091272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477091272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477091281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/30e602c490624a25a642024ba0345e4f, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T05:50:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d6ecd69dbfe14439b0666324f69d9576 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d6ecd69dbfe14439b0666324f69d9576 2024-11-13T05:50:31,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d6ecd69dbfe14439b0666324f69d9576, entries=150, sequenceid=144, filesize=11.9 K 2024-11-13T05:50:31,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 9480a359b149cb565364fd0f1f82db91 in 737ms, sequenceid=144, compaction requested=true 2024-11-13T05:50:31,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:50:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T05:50:31,315 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:31,315 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:31,324 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:31,324 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:31,324 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:31,325 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d9ae103b09ec42179089bc1b4a3962ae, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/735fcc2a96094f938973992eea520ff3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e631cabbd84a495b930f500baba07da1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0e84963a6df24bdb9e22cc2d092b2a69] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=49.4 K 2024-11-13T05:50:31,325 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:31,325 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:31,325 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:31,325 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5ca7980182de4c459eb7abbd55c0cff1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/7b1f0fd615224c6b8a91e7a45efeaa17, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/086166cb28984d5789328122b700b223, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d6ecd69dbfe14439b0666324f69d9576] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=47.2 K 2024-11-13T05:50:31,326 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca7980182de4c459eb7abbd55c0cff1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731477026120 2024-11-13T05:50:31,326 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ae103b09ec42179089bc1b4a3962ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731477026120 2024-11-13T05:50:31,327 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735fcc2a96094f938973992eea520ff3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731477026218 2024-11-13T05:50:31,328 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e631cabbd84a495b930f500baba07da1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477027483 2024-11-13T05:50:31,329 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1f0fd615224c6b8a91e7a45efeaa17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731477026237 2024-11-13T05:50:31,330 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e84963a6df24bdb9e22cc2d092b2a69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731477029898 2024-11-13T05:50:31,330 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 086166cb28984d5789328122b700b223, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477027483 2024-11-13T05:50:31,333 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ecd69dbfe14439b0666324f69d9576, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731477029898 2024-11-13T05:50:31,364 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:31,365 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1ad92fe2b4d240568fb2580bad240cf1 is 50, key is test_row_0/A:col10/1731477030568/Put/seqid=0 2024-11-13T05:50:31,369 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:31,370 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/85179127be49468a940d4279d85753cb is 50, key is test_row_0/C:col10/1731477030568/Put/seqid=0 2024-11-13T05:50:31,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741862_1038 (size=12425) 2024-11-13T05:50:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741861_1037 (size=12425) 2024-11-13T05:50:31,432 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1ad92fe2b4d240568fb2580bad240cf1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1ad92fe2b4d240568fb2580bad240cf1 2024-11-13T05:50:31,447 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into 1ad92fe2b4d240568fb2580bad240cf1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:31,447 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:31,447 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=12, startTime=1731477031314; duration=0sec 2024-11-13T05:50:31,447 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:31,447 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:31,447 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:31,450 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:31,450 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:31,450 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:31,450 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/277ed23bd572469690923c228eebe18d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1c504ae1e6184d7aa51f7c7c1152945f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/829eb1a2551544058e98bd4fb3f17cf6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/30e602c490624a25a642024ba0345e4f] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=47.2 K 2024-11-13T05:50:31,451 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 277ed23bd572469690923c228eebe18d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731477026120 2024-11-13T05:50:31,452 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c504ae1e6184d7aa51f7c7c1152945f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731477026237 2024-11-13T05:50:31,453 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 829eb1a2551544058e98bd4fb3f17cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477027483 2024-11-13T05:50:31,454 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30e602c490624a25a642024ba0345e4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731477029898 2024-11-13T05:50:31,490 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#26 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:31,491 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1d0ea2acf02c4ea49220f30d64df91f6 is 50, key is test_row_0/B:col10/1731477030568/Put/seqid=0 2024-11-13T05:50:31,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741863_1039 (size=12425) 2024-11-13T05:50:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:31,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:50:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:31,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/97d5713497ea48c9a2342a55ab9657b4 is 50, key is test_row_0/A:col10/1731477030602/Put/seqid=0 2024-11-13T05:50:31,831 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/85179127be49468a940d4279d85753cb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/85179127be49468a940d4279d85753cb 2024-11-13T05:50:31,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477091816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477091834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477091836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477091843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477091843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741864_1040 (size=14541) 2024-11-13T05:50:31,849 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into 85179127be49468a940d4279d85753cb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:31,849 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:31,849 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=12, startTime=1731477031315; duration=0sec 2024-11-13T05:50:31,849 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:31,850 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:31,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/97d5713497ea48c9a2342a55ab9657b4 2024-11-13T05:50:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/7c0ed1ecfc474cfa93f247ab496765f5 is 50, key is test_row_0/B:col10/1731477030602/Put/seqid=0 2024-11-13T05:50:31,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741865_1041 (size=12151) 2024-11-13T05:50:31,933 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/1d0ea2acf02c4ea49220f30d64df91f6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1d0ea2acf02c4ea49220f30d64df91f6 2024-11-13T05:50:31,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477091942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,947 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 1d0ea2acf02c4ea49220f30d64df91f6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:31,947 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:31,947 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=12, startTime=1731477031315; duration=0sec 2024-11-13T05:50:31,948 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:31,948 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477091952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477091952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477091952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:31,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477091971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T05:50:32,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:32,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T05:50:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:50:32,086 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:32,088 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:32,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:32,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477092151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477092163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477092163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477092163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477092182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:50:32,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:32,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:32,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/7c0ed1ecfc474cfa93f247ab496765f5 2024-11-13T05:50:32,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/ede10a9d753144b593f015eeb710c5d9 is 50, key is test_row_0/C:col10/1731477030602/Put/seqid=0 2024-11-13T05:50:32,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741866_1042 (size=12151) 2024-11-13T05:50:32,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:32,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:32,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:50:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477092462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477092471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477092471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477092471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477092502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:32,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:32,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:32,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:50:32,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:32,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/ede10a9d753144b593f015eeb710c5d9 2024-11-13T05:50:32,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/97d5713497ea48c9a2342a55ab9657b4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/97d5713497ea48c9a2342a55ab9657b4 2024-11-13T05:50:32,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/97d5713497ea48c9a2342a55ab9657b4, entries=200, sequenceid=165, filesize=14.2 K 2024-11-13T05:50:32,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/7c0ed1ecfc474cfa93f247ab496765f5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/7c0ed1ecfc474cfa93f247ab496765f5 2024-11-13T05:50:32,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/7c0ed1ecfc474cfa93f247ab496765f5, entries=150, sequenceid=165, filesize=11.9 K 2024-11-13T05:50:32,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/ede10a9d753144b593f015eeb710c5d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/ede10a9d753144b593f015eeb710c5d9 2024-11-13T05:50:32,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:32,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:32,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/ede10a9d753144b593f015eeb710c5d9, entries=150, sequenceid=165, filesize=11.9 K 2024-11-13T05:50:32,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9480a359b149cb565364fd0f1f82db91 in 1103ms, sequenceid=165, compaction requested=false 2024-11-13T05:50:32,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:32,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:50:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:32,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:32,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:32,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:32,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9cc2f02b926a4de1901a779f6013546b is 50, key is test_row_0/A:col10/1731477031832/Put/seqid=0 2024-11-13T05:50:33,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741867_1043 (size=12151) 2024-11-13T05:50:33,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9cc2f02b926a4de1901a779f6013546b 2024-11-13T05:50:33,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:33,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:33,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477093040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477093041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477093043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477093056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477093056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/e8acfd920e9f468580bd8cf8ec74c0a2 is 50, key is test_row_0/B:col10/1731477031832/Put/seqid=0 2024-11-13T05:50:33,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741868_1044 (size=12151) 2024-11-13T05:50:33,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/e8acfd920e9f468580bd8cf8ec74c0a2 2024-11-13T05:50:33,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/3b883becb99f48a18c9bb9d213efedfa is 50, key is test_row_0/C:col10/1731477031832/Put/seqid=0 2024-11-13T05:50:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477093161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477093162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477093162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477093171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477093171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741869_1045 (size=12151) 2024-11-13T05:50:33,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/3b883becb99f48a18c9bb9d213efedfa 2024-11-13T05:50:33,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9cc2f02b926a4de1901a779f6013546b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9cc2f02b926a4de1901a779f6013546b 2024-11-13T05:50:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:33,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9cc2f02b926a4de1901a779f6013546b, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T05:50:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/e8acfd920e9f468580bd8cf8ec74c0a2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8acfd920e9f468580bd8cf8ec74c0a2 2024-11-13T05:50:33,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8acfd920e9f468580bd8cf8ec74c0a2, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T05:50:33,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/3b883becb99f48a18c9bb9d213efedfa as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/3b883becb99f48a18c9bb9d213efedfa 2024-11-13T05:50:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:50:33,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/3b883becb99f48a18c9bb9d213efedfa, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T05:50:33,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 9480a359b149cb565364fd0f1f82db91 in 258ms, sequenceid=180, compaction requested=true 2024-11-13T05:50:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:33,232 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:33,232 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:33,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:33,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:33,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:33,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:33,234 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:33,234 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:33,234 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,235 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1ad92fe2b4d240568fb2580bad240cf1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/97d5713497ea48c9a2342a55ab9657b4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9cc2f02b926a4de1901a779f6013546b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=38.2 K 2024-11-13T05:50:33,235 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:33,235 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:33,235 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,235 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1d0ea2acf02c4ea49220f30d64df91f6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/7c0ed1ecfc474cfa93f247ab496765f5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8acfd920e9f468580bd8cf8ec74c0a2] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=35.9 K 2024-11-13T05:50:33,236 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ad92fe2b4d240568fb2580bad240cf1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731477029898 2024-11-13T05:50:33,236 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0ea2acf02c4ea49220f30d64df91f6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731477029898 2024-11-13T05:50:33,237 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97d5713497ea48c9a2342a55ab9657b4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731477030602 2024-11-13T05:50:33,237 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0ed1ecfc474cfa93f247ab496765f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731477030602 2024-11-13T05:50:33,239 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc2f02b926a4de1901a779f6013546b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477031807 2024-11-13T05:50:33,241 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e8acfd920e9f468580bd8cf8ec74c0a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477031807 2024-11-13T05:50:33,312 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#33 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:33,312 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/d48430f766a34fb79eb554cf7590ac14 is 50, key is test_row_0/A:col10/1731477031832/Put/seqid=0 2024-11-13T05:50:33,348 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#34 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:33,348 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/9839eabd0f794958805f38ad4bed6331 is 50, key is test_row_0/B:col10/1731477031832/Put/seqid=0 2024-11-13T05:50:33,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T05:50:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,351 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741870_1046 (size=12527) 2024-11-13T05:50:33,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/895041719e424a94ae5a7336f1ac410b is 50, key is test_row_0/A:col10/1731477033053/Put/seqid=0 2024-11-13T05:50:33,382 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/d48430f766a34fb79eb554cf7590ac14 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d48430f766a34fb79eb554cf7590ac14 2024-11-13T05:50:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:33,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:33,397 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into d48430f766a34fb79eb554cf7590ac14(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:33,397 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:33,397 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=13, startTime=1731477033232; duration=0sec 2024-11-13T05:50:33,398 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:33,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:33,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:33,402 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:33,402 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:33,402 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,402 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/85179127be49468a940d4279d85753cb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/ede10a9d753144b593f015eeb710c5d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/3b883becb99f48a18c9bb9d213efedfa] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=35.9 K 2024-11-13T05:50:33,404 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85179127be49468a940d4279d85753cb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731477029898 2024-11-13T05:50:33,406 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede10a9d753144b593f015eeb710c5d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731477030602 2024-11-13T05:50:33,408 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b883becb99f48a18c9bb9d213efedfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477031807 2024-11-13T05:50:33,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741871_1047 (size=12527) 2024-11-13T05:50:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477093404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,425 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/9839eabd0f794958805f38ad4bed6331 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/9839eabd0f794958805f38ad4bed6331 2024-11-13T05:50:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477093408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477093411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477093414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477093415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,439 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 9839eabd0f794958805f38ad4bed6331(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:33,440 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:33,440 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=13, startTime=1731477033232; duration=0sec 2024-11-13T05:50:33,440 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:33,440 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:33,440 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:33,440 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5db667aba8b94098a087bc9aa4664fa5 is 50, key is test_row_0/C:col10/1731477031832/Put/seqid=0 2024-11-13T05:50:33,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741872_1048 (size=12151) 2024-11-13T05:50:33,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/895041719e424a94ae5a7336f1ac410b 2024-11-13T05:50:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/60bf0476484a401a971a6100b5cefa9d is 50, key is test_row_0/B:col10/1731477033053/Put/seqid=0 2024-11-13T05:50:33,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741873_1049 (size=12527) 2024-11-13T05:50:33,534 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5db667aba8b94098a087bc9aa4664fa5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5db667aba8b94098a087bc9aa4664fa5 2024-11-13T05:50:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477093532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477093541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477093542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477093542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,550 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into 5db667aba8b94098a087bc9aa4664fa5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:33,550 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:33,550 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=13, startTime=1731477033233; duration=0sec 2024-11-13T05:50:33,551 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:33,551 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477093548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741874_1050 (size=12151) 2024-11-13T05:50:33,571 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/60bf0476484a401a971a6100b5cefa9d 2024-11-13T05:50:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/477d6468ec4f465ba682d67e971296ff is 50, key is test_row_0/C:col10/1731477033053/Put/seqid=0 2024-11-13T05:50:33,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741875_1051 (size=12151) 2024-11-13T05:50:33,614 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/477d6468ec4f465ba682d67e971296ff 2024-11-13T05:50:33,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/895041719e424a94ae5a7336f1ac410b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/895041719e424a94ae5a7336f1ac410b 2024-11-13T05:50:33,637 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/895041719e424a94ae5a7336f1ac410b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-13T05:50:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/60bf0476484a401a971a6100b5cefa9d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/60bf0476484a401a971a6100b5cefa9d 2024-11-13T05:50:33,651 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/60bf0476484a401a971a6100b5cefa9d, entries=150, sequenceid=200, filesize=11.9 K 2024-11-13T05:50:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/477d6468ec4f465ba682d67e971296ff as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/477d6468ec4f465ba682d67e971296ff 2024-11-13T05:50:33,661 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/477d6468ec4f465ba682d67e971296ff, entries=150, sequenceid=200, filesize=11.9 K 2024-11-13T05:50:33,664 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9480a359b149cb565364fd0f1f82db91 in 313ms, sequenceid=200, compaction requested=false 2024-11-13T05:50:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:33,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:33,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T05:50:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T05:50:33,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T05:50:33,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5790 sec 2024-11-13T05:50:33,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5920 sec 2024-11-13T05:50:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:33,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:50:33,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:33,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:33,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:33,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:33,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:33,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:33,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/b9335048a5d5452a85ff8961e927e2eb is 50, key is test_row_0/A:col10/1731477033413/Put/seqid=0 2024-11-13T05:50:33,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741876_1052 (size=12151) 2024-11-13T05:50:33,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/b9335048a5d5452a85ff8961e927e2eb 2024-11-13T05:50:33,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ac8cc5a97c6e487fb9e589bb4816d4f5 is 50, key is test_row_0/B:col10/1731477033413/Put/seqid=0 2024-11-13T05:50:33,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741877_1053 (size=12151) 2024-11-13T05:50:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477093848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477093859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477093861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477093897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:33,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477093908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477094012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477094012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477094013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477094013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477094021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477094222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T05:50:34,233 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:34,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:34,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477094232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T05:50:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477094233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,240 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:50:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477094233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,241 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:34,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477094233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ac8cc5a97c6e487fb9e589bb4816d4f5 2024-11-13T05:50:34,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5d38b2eef0214e8fb1eb081eff4e22c6 is 50, key is test_row_0/C:col10/1731477033413/Put/seqid=0 2024-11-13T05:50:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:50:34,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741878_1054 (size=12151) 2024-11-13T05:50:34,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5d38b2eef0214e8fb1eb081eff4e22c6 2024-11-13T05:50:34,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/b9335048a5d5452a85ff8961e927e2eb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9335048a5d5452a85ff8961e927e2eb 2024-11-13T05:50:34,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9335048a5d5452a85ff8961e927e2eb, entries=150, sequenceid=218, filesize=11.9 K 2024-11-13T05:50:34,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ac8cc5a97c6e487fb9e589bb4816d4f5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ac8cc5a97c6e487fb9e589bb4816d4f5 2024-11-13T05:50:34,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ac8cc5a97c6e487fb9e589bb4816d4f5, entries=150, sequenceid=218, filesize=11.9 K 2024-11-13T05:50:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/5d38b2eef0214e8fb1eb081eff4e22c6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5d38b2eef0214e8fb1eb081eff4e22c6 2024-11-13T05:50:34,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:50:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5d38b2eef0214e8fb1eb081eff4e22c6, entries=150, sequenceid=218, filesize=11.9 K 2024-11-13T05:50:34,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 9480a359b149cb565364fd0f1f82db91 in 612ms, sequenceid=218, compaction requested=true 2024-11-13T05:50:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:34,405 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:34,405 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:34,407 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:34,408 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:34,408 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,408 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/9839eabd0f794958805f38ad4bed6331, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/60bf0476484a401a971a6100b5cefa9d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ac8cc5a97c6e487fb9e589bb4816d4f5] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.0 K 2024-11-13T05:50:34,408 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:34,408 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:34,408 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,408 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9839eabd0f794958805f38ad4bed6331, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477031807 2024-11-13T05:50:34,408 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d48430f766a34fb79eb554cf7590ac14, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/895041719e424a94ae5a7336f1ac410b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9335048a5d5452a85ff8961e927e2eb] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.0 K 2024-11-13T05:50:34,409 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48430f766a34fb79eb554cf7590ac14, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477031807 2024-11-13T05:50:34,409 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bf0476484a401a971a6100b5cefa9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731477033036 2024-11-13T05:50:34,410 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895041719e424a94ae5a7336f1ac410b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731477033036 2024-11-13T05:50:34,410 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8cc5a97c6e487fb9e589bb4816d4f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731477033410 2024-11-13T05:50:34,410 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9335048a5d5452a85ff8961e927e2eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731477033410 2024-11-13T05:50:34,432 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:34,433 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ad7bef3f698049dfb2add9a0abb734d9 is 50, key is test_row_0/B:col10/1731477033413/Put/seqid=0 2024-11-13T05:50:34,442 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:34,443 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1a51ff7f19494beea6f8bfd7839300bf is 50, key is test_row_0/A:col10/1731477033413/Put/seqid=0 2024-11-13T05:50:34,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741879_1055 (size=12629) 2024-11-13T05:50:34,478 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ad7bef3f698049dfb2add9a0abb734d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7bef3f698049dfb2add9a0abb734d9 2024-11-13T05:50:34,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741880_1056 (size=12629) 2024-11-13T05:50:34,491 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into ad7bef3f698049dfb2add9a0abb734d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:34,491 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:34,491 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=13, startTime=1731477034405; duration=0sec 2024-11-13T05:50:34,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:34,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:34,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:34,496 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:34,497 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:34,498 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,498 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5db667aba8b94098a087bc9aa4664fa5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/477d6468ec4f465ba682d67e971296ff, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5d38b2eef0214e8fb1eb081eff4e22c6] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.0 K 2024-11-13T05:50:34,499 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5db667aba8b94098a087bc9aa4664fa5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477031807 2024-11-13T05:50:34,500 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 477d6468ec4f465ba682d67e971296ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731477033036 2024-11-13T05:50:34,501 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1a51ff7f19494beea6f8bfd7839300bf as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1a51ff7f19494beea6f8bfd7839300bf 2024-11-13T05:50:34,501 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d38b2eef0214e8fb1eb081eff4e22c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731477033410 2024-11-13T05:50:34,520 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into 1a51ff7f19494beea6f8bfd7839300bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:34,520 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:34,520 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=13, startTime=1731477034405; duration=0sec 2024-11-13T05:50:34,520 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:34,520 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:34,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T05:50:34,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:34,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/0a66b6c502c24fb9ac80e9d727015331 is 50, key is test_row_0/A:col10/1731477034533/Put/seqid=0 2024-11-13T05:50:34,547 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:34,548 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/eb764321e567415ba84c103d6c0a2193 is 50, key is test_row_0/C:col10/1731477033413/Put/seqid=0 2024-11-13T05:50:34,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:50:34,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741881_1057 (size=14541) 2024-11-13T05:50:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:34,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/0a66b6c502c24fb9ac80e9d727015331 2024-11-13T05:50:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:50:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477094569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/4bdb94c5dc6e4121b92c5f39ce3090c3 is 50, key is test_row_0/B:col10/1731477034533/Put/seqid=0 2024-11-13T05:50:34,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477094573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741882_1058 (size=12629) 2024-11-13T05:50:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477094575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477094577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477094577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,598 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/eb764321e567415ba84c103d6c0a2193 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/eb764321e567415ba84c103d6c0a2193 2024-11-13T05:50:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741883_1059 (size=12151) 2024-11-13T05:50:34,617 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into eb764321e567415ba84c103d6c0a2193(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:34,617 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:34,618 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=13, startTime=1731477034405; duration=0sec 2024-11-13T05:50:34,618 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:34,618 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:34,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477094681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477094698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477094698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477094702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:50:34,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477094702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:50:34,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:34,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:34,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:50:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:34,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477094892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477094922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477094923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477094923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477094923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/4bdb94c5dc6e4121b92c5f39ce3090c3 2024-11-13T05:50:35,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:50:35,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:35,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:35,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:35,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fcc3fed020434ec497c884f0042e3c30 is 50, key is test_row_0/C:col10/1731477034533/Put/seqid=0 2024-11-13T05:50:35,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741884_1060 (size=12151) 2024-11-13T05:50:35,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fcc3fed020434ec497c884f0042e3c30 2024-11-13T05:50:35,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/0a66b6c502c24fb9ac80e9d727015331 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0a66b6c502c24fb9ac80e9d727015331 2024-11-13T05:50:35,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0a66b6c502c24fb9ac80e9d727015331, entries=200, sequenceid=239, filesize=14.2 K 2024-11-13T05:50:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/4bdb94c5dc6e4121b92c5f39ce3090c3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4bdb94c5dc6e4121b92c5f39ce3090c3 2024-11-13T05:50:35,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4bdb94c5dc6e4121b92c5f39ce3090c3, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T05:50:35,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fcc3fed020434ec497c884f0042e3c30 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fcc3fed020434ec497c884f0042e3c30 2024-11-13T05:50:35,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fcc3fed020434ec497c884f0042e3c30, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T05:50:35,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 9480a359b149cb565364fd0f1f82db91 in 616ms, sequenceid=239, compaction requested=false 2024-11-13T05:50:35,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T05:50:35,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,182 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:50:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:35,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/36ac3bb587994f179e144b3e69d125d9 is 50, key is test_row_0/A:col10/1731477034572/Put/seqid=0 2024-11-13T05:50:35,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741885_1061 (size=12151) 2024-11-13T05:50:35,211 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/36ac3bb587994f179e144b3e69d125d9 2024-11-13T05:50:35,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:35,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/51901d7d91f6401bb788942200414ab6 is 50, key is test_row_0/B:col10/1731477034572/Put/seqid=0 2024-11-13T05:50:35,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741886_1062 (size=12151) 2024-11-13T05:50:35,258 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/51901d7d91f6401bb788942200414ab6 2024-11-13T05:50:35,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/a83e3b874d6148b49b4c16618896a054 is 50, key is test_row_0/C:col10/1731477034572/Put/seqid=0 2024-11-13T05:50:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741887_1063 (size=12151) 2024-11-13T05:50:35,313 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/a83e3b874d6148b49b4c16618896a054 2024-11-13T05:50:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/36ac3bb587994f179e144b3e69d125d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/36ac3bb587994f179e144b3e69d125d9 2024-11-13T05:50:35,332 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/36ac3bb587994f179e144b3e69d125d9, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T05:50:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/51901d7d91f6401bb788942200414ab6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/51901d7d91f6401bb788942200414ab6 2024-11-13T05:50:35,341 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/51901d7d91f6401bb788942200414ab6, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T05:50:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/a83e3b874d6148b49b4c16618896a054 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/a83e3b874d6148b49b4c16618896a054 2024-11-13T05:50:35,356 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/a83e3b874d6148b49b4c16618896a054, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T05:50:35,365 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=80.51 KB/82440 for 9480a359b149cb565364fd0f1f82db91 in 183ms, sequenceid=254, compaction requested=true 2024-11-13T05:50:35,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T05:50:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T05:50:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:35,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:50:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T05:50:35,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1280 sec 2024-11-13T05:50:35,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1400 sec 2024-11-13T05:50:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T05:50:35,382 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:35,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/a7b6a48728fe41c5be3caf3422beca56 is 50, key is test_row_0/A:col10/1731477035301/Put/seqid=0 2024-11-13T05:50:35,386 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T05:50:35,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:50:35,390 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:35,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:35,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741888_1064 (size=17181) 2024-11-13T05:50:35,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477095410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/a7b6a48728fe41c5be3caf3422beca56 2024-11-13T05:50:35,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477095413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477095415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477095434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477095434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/e8b7fb6f809740fbad74d1fec99e750a is 50, key is test_row_0/B:col10/1731477035301/Put/seqid=0 2024-11-13T05:50:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:50:35,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741889_1065 (size=12301) 2024-11-13T05:50:35,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/e8b7fb6f809740fbad74d1fec99e750a 2024-11-13T05:50:35,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1739b97f2fb848a79516ef482fb4b73d is 50, key is test_row_0/C:col10/1731477035301/Put/seqid=0 2024-11-13T05:50:35,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477095542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477095542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477095542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:50:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:35,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:35,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:35,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477095553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477095554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741890_1066 (size=12301) 2024-11-13T05:50:35,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1739b97f2fb848a79516ef482fb4b73d 2024-11-13T05:50:35,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/a7b6a48728fe41c5be3caf3422beca56 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a7b6a48728fe41c5be3caf3422beca56 2024-11-13T05:50:35,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a7b6a48728fe41c5be3caf3422beca56, entries=250, sequenceid=270, filesize=16.8 K 2024-11-13T05:50:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/e8b7fb6f809740fbad74d1fec99e750a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8b7fb6f809740fbad74d1fec99e750a 2024-11-13T05:50:35,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8b7fb6f809740fbad74d1fec99e750a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-13T05:50:35,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1739b97f2fb848a79516ef482fb4b73d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1739b97f2fb848a79516ef482fb4b73d 2024-11-13T05:50:35,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1739b97f2fb848a79516ef482fb4b73d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-13T05:50:35,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 9480a359b149cb565364fd0f1f82db91 in 267ms, sequenceid=270, compaction requested=true 2024-11-13T05:50:35,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,634 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:35,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:35,637 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:35,638 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56502 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:35,638 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:35,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:35,638 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:35,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:35,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:35,638 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1a51ff7f19494beea6f8bfd7839300bf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0a66b6c502c24fb9ac80e9d727015331, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/36ac3bb587994f179e144b3e69d125d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a7b6a48728fe41c5be3caf3422beca56] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=55.2 K 2024-11-13T05:50:35,639 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a51ff7f19494beea6f8bfd7839300bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731477033410 2024-11-13T05:50:35,639 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:35,639 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a66b6c502c24fb9ac80e9d727015331, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731477033846 2024-11-13T05:50:35,639 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:35,639 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,639 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7bef3f698049dfb2add9a0abb734d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4bdb94c5dc6e4121b92c5f39ce3090c3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/51901d7d91f6401bb788942200414ab6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8b7fb6f809740fbad74d1fec99e750a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=48.1 K 2024-11-13T05:50:35,640 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ac3bb587994f179e144b3e69d125d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731477034572 2024-11-13T05:50:35,640 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ad7bef3f698049dfb2add9a0abb734d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731477033410 2024-11-13T05:50:35,640 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b6a48728fe41c5be3caf3422beca56, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731477035248 2024-11-13T05:50:35,641 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bdb94c5dc6e4121b92c5f39ce3090c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731477033846 2024-11-13T05:50:35,642 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 51901d7d91f6401bb788942200414ab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731477034572 2024-11-13T05:50:35,642 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b7fb6f809740fbad74d1fec99e750a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731477035301 2024-11-13T05:50:35,665 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:35,666 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/c9df3410c33545919f7d962757a46c54 is 50, key is test_row_0/A:col10/1731477035301/Put/seqid=0 2024-11-13T05:50:35,669 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:35,670 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61 is 50, key is test_row_0/B:col10/1731477035301/Put/seqid=0 2024-11-13T05:50:35,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741891_1067 (size=12915) 2024-11-13T05:50:35,699 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/c9df3410c33545919f7d962757a46c54 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/c9df3410c33545919f7d962757a46c54 2024-11-13T05:50:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:50:35,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741892_1068 (size=12915) 2024-11-13T05:50:35,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T05:50:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,720 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:50:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:35,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:35,721 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into c9df3410c33545919f7d962757a46c54(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:35,721 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61 2024-11-13T05:50:35,721 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,722 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=12, startTime=1731477035634; duration=0sec 2024-11-13T05:50:35,722 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:35,722 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:35,722 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:35,725 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:35,725 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:35,726 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,732 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into dff9ed05534f4d7cbf5a3d5e6b4a3b61(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:35,732 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,733 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=12, startTime=1731477035637; duration=0sec 2024-11-13T05:50:35,733 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:35,733 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:35,733 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/eb764321e567415ba84c103d6c0a2193, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fcc3fed020434ec497c884f0042e3c30, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/a83e3b874d6148b49b4c16618896a054, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1739b97f2fb848a79516ef482fb4b73d] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=48.1 K 2024-11-13T05:50:35,736 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb764321e567415ba84c103d6c0a2193, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731477033410 2024-11-13T05:50:35,737 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc3fed020434ec497c884f0042e3c30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731477033846 2024-11-13T05:50:35,737 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a83e3b874d6148b49b4c16618896a054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731477034572 2024-11-13T05:50:35,738 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1739b97f2fb848a79516ef482fb4b73d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731477035301 2024-11-13T05:50:35,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/647b087db86349b598a9a5199c8e1618 is 50, key is test_row_0/A:col10/1731477035425/Put/seqid=0 2024-11-13T05:50:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:35,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:35,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741893_1069 (size=12301) 2024-11-13T05:50:35,784 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/647b087db86349b598a9a5199c8e1618 2024-11-13T05:50:35,785 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:35,786 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/814f5ee504514f40a44895c3da248781 is 50, key is test_row_0/C:col10/1731477035301/Put/seqid=0 2024-11-13T05:50:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ef038ec71aa14036b356ba430cc7373c is 50, key is test_row_0/B:col10/1731477035425/Put/seqid=0 2024-11-13T05:50:35,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477095793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477095796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741894_1070 (size=12915) 2024-11-13T05:50:35,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477095797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477095790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477095811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,823 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/814f5ee504514f40a44895c3da248781 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/814f5ee504514f40a44895c3da248781 2024-11-13T05:50:35,833 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into 814f5ee504514f40a44895c3da248781(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:35,833 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,833 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=12, startTime=1731477035638; duration=0sec 2024-11-13T05:50:35,833 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:35,833 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:35,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741895_1071 (size=12301) 2024-11-13T05:50:35,845 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ef038ec71aa14036b356ba430cc7373c 2024-11-13T05:50:35,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/250d3bc15b564b9c8883a13c2bba5fca is 50, key is test_row_0/C:col10/1731477035425/Put/seqid=0 2024-11-13T05:50:35,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741896_1072 (size=12301) 2024-11-13T05:50:35,918 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/250d3bc15b564b9c8883a13c2bba5fca 2024-11-13T05:50:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/647b087db86349b598a9a5199c8e1618 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/647b087db86349b598a9a5199c8e1618 2024-11-13T05:50:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477095923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477095923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477095923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477095923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477095933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:35,940 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/647b087db86349b598a9a5199c8e1618, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T05:50:35,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ef038ec71aa14036b356ba430cc7373c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ef038ec71aa14036b356ba430cc7373c 2024-11-13T05:50:35,949 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ef038ec71aa14036b356ba430cc7373c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T05:50:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/250d3bc15b564b9c8883a13c2bba5fca as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/250d3bc15b564b9c8883a13c2bba5fca 2024-11-13T05:50:35,962 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/250d3bc15b564b9c8883a13c2bba5fca, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T05:50:35,964 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9480a359b149cb565364fd0f1f82db91 in 243ms, sequenceid=288, compaction requested=false 2024-11-13T05:50:35,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:35,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:35,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T05:50:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T05:50:35,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T05:50:35,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 576 msec 2024-11-13T05:50:35,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 587 msec 2024-11-13T05:50:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T05:50:36,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:36,013 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T05:50:36,034 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:50:36,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:36,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:50:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:36,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:50:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:36,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/28c9e47347eb4d708c495a14dcf88acd is 50, key is test_row_0/A:col10/1731477035789/Put/seqid=0 2024-11-13T05:50:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477096185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477096185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477096194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477096194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:50:36,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477096198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:36,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741897_1073 (size=12301) 2024-11-13T05:50:36,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/28c9e47347eb4d708c495a14dcf88acd 2024-11-13T05:50:36,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/71d2db42c1de41cc828a8205132b11f8 is 50, key is test_row_0/B:col10/1731477035789/Put/seqid=0 2024-11-13T05:50:36,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741898_1074 (size=12301) 2024-11-13T05:50:36,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/71d2db42c1de41cc828a8205132b11f8 2024-11-13T05:50:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477096302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477096302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/e97c1d27f7bf47b3900c3c313e6989c7 is 50, key is test_row_0/C:col10/1731477035789/Put/seqid=0 2024-11-13T05:50:36,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477096313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477096313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477096313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741899_1075 (size=12301) 2024-11-13T05:50:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:50:36,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:50:36,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:50:36,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:36,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477096518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477096522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477096522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477096522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477096518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:50:36,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:50:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:36,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/e97c1d27f7bf47b3900c3c313e6989c7 2024-11-13T05:50:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/28c9e47347eb4d708c495a14dcf88acd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/28c9e47347eb4d708c495a14dcf88acd 2024-11-13T05:50:36,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/28c9e47347eb4d708c495a14dcf88acd, entries=150, sequenceid=308, filesize=12.0 K 2024-11-13T05:50:36,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/71d2db42c1de41cc828a8205132b11f8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/71d2db42c1de41cc828a8205132b11f8 2024-11-13T05:50:36,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/71d2db42c1de41cc828a8205132b11f8, entries=150, sequenceid=308, filesize=12.0 K 2024-11-13T05:50:36,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/e97c1d27f7bf47b3900c3c313e6989c7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/e97c1d27f7bf47b3900c3c313e6989c7 2024-11-13T05:50:36,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/e97c1d27f7bf47b3900c3c313e6989c7, entries=150, sequenceid=308, filesize=12.0 K 2024-11-13T05:50:36,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9480a359b149cb565364fd0f1f82db91 in 669ms, sequenceid=308, compaction requested=true 2024-11-13T05:50:36,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:36,824 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:50:36,825 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:36,825 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:36,825 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:36,825 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,826 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/c9df3410c33545919f7d962757a46c54, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/647b087db86349b598a9a5199c8e1618, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/28c9e47347eb4d708c495a14dcf88acd] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.6 K 2024-11-13T05:50:36,826 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:36,826 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:36,826 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,826 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ef038ec71aa14036b356ba430cc7373c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/71d2db42c1de41cc828a8205132b11f8] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.6 K 2024-11-13T05:50:36,826 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9df3410c33545919f7d962757a46c54, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731477035301 2024-11-13T05:50:36,826 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting dff9ed05534f4d7cbf5a3d5e6b4a3b61, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731477035301 2024-11-13T05:50:36,827 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647b087db86349b598a9a5199c8e1618, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731477035402 2024-11-13T05:50:36,827 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ef038ec71aa14036b356ba430cc7373c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731477035402 2024-11-13T05:50:36,827 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d2db42c1de41cc828a8205132b11f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731477035789 2024-11-13T05:50:36,827 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c9e47347eb4d708c495a14dcf88acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731477035789 2024-11-13T05:50:36,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T05:50:36,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,838 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:50:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:36,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:36,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:36,856 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:36,858 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/2baf06d5a9a549e0acb05b260a56db13 is 50, key is test_row_0/B:col10/1731477035789/Put/seqid=0 2024-11-13T05:50:36,873 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:36,873 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/8fba0cb100db42579e32f1a923b247dd is 50, key is test_row_0/A:col10/1731477035789/Put/seqid=0 2024-11-13T05:50:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/62bd712a5e4443b99ab327b51e1ad2c0 is 50, key is test_row_0/A:col10/1731477036186/Put/seqid=0 2024-11-13T05:50:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477096892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477096889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477096913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477096914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477096916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:36,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741900_1076 (size=13017) 2024-11-13T05:50:36,959 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/2baf06d5a9a549e0acb05b260a56db13 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2baf06d5a9a549e0acb05b260a56db13 2024-11-13T05:50:36,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741901_1077 (size=13017) 2024-11-13T05:50:36,978 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 2baf06d5a9a549e0acb05b260a56db13(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:36,978 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:36,978 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=13, startTime=1731477036824; duration=0sec 2024-11-13T05:50:36,979 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:36,979 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:36,979 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:36,987 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:36,987 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:36,988 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:36,988 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/814f5ee504514f40a44895c3da248781, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/250d3bc15b564b9c8883a13c2bba5fca, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/e97c1d27f7bf47b3900c3c313e6989c7] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.6 K 2024-11-13T05:50:36,992 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/8fba0cb100db42579e32f1a923b247dd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/8fba0cb100db42579e32f1a923b247dd 2024-11-13T05:50:36,993 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 814f5ee504514f40a44895c3da248781, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731477035301 2024-11-13T05:50:36,995 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 250d3bc15b564b9c8883a13c2bba5fca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731477035402 2024-11-13T05:50:36,997 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e97c1d27f7bf47b3900c3c313e6989c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731477035789 2024-11-13T05:50:37,004 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into 8fba0cb100db42579e32f1a923b247dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:37,004 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:37,004 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=13, startTime=1731477036823; duration=0sec 2024-11-13T05:50:37,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741902_1078 (size=12301) 2024-11-13T05:50:37,004 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:37,004 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:37,005 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/62bd712a5e4443b99ab327b51e1ad2c0 2024-11-13T05:50:37,026 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:37,027 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/b7a167c71c134246924a032cfd66e3f9 is 50, key is test_row_0/C:col10/1731477035789/Put/seqid=0 2024-11-13T05:50:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477097022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477097032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477097032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/164d76250bc44e75b8c9f17e8d62208b is 50, key is test_row_0/B:col10/1731477036186/Put/seqid=0 2024-11-13T05:50:37,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477097042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741903_1079 (size=13017) 2024-11-13T05:50:37,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741904_1080 (size=12301) 2024-11-13T05:50:37,139 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/164d76250bc44e75b8c9f17e8d62208b 2024-11-13T05:50:37,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/4eaf8f3c962348cf88ab1f87eca25c81 is 50, key is test_row_0/C:col10/1731477036186/Put/seqid=0 2024-11-13T05:50:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:50:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741905_1081 (size=12301) 2024-11-13T05:50:37,214 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/4eaf8f3c962348cf88ab1f87eca25c81 2024-11-13T05:50:37,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/62bd712a5e4443b99ab327b51e1ad2c0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/62bd712a5e4443b99ab327b51e1ad2c0 2024-11-13T05:50:37,244 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/62bd712a5e4443b99ab327b51e1ad2c0, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T05:50:37,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/164d76250bc44e75b8c9f17e8d62208b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/164d76250bc44e75b8c9f17e8d62208b 2024-11-13T05:50:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477097242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477097252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477097254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477097254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,260 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/164d76250bc44e75b8c9f17e8d62208b, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T05:50:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/4eaf8f3c962348cf88ab1f87eca25c81 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/4eaf8f3c962348cf88ab1f87eca25c81 2024-11-13T05:50:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,273 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/4eaf8f3c962348cf88ab1f87eca25c81, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T05:50:37,275 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 9480a359b149cb565364fd0f1f82db91 in 437ms, sequenceid=322, compaction requested=false 2024-11-13T05:50:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T05:50:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T05:50:37,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T05:50:37,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2024-11-13T05:50:37,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2750 sec 2024-11-13T05:50:37,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477036878, max timestamp=1731477037277Num uniques 859; 2024-11-13T05:50:37,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:37,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-13T05:50:37,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:37,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:37,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:37,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/3c76b052ae9b4de28248cad6a52c7070 is 50, key is test_row_0/A:col10/1731477037277/Put/seqid=0 2024-11-13T05:50:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741906_1082 (size=14741) 2024-11-13T05:50:37,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/3c76b052ae9b4de28248cad6a52c7070 2024-11-13T05:50:37,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/d03335862ef04362a035f2eb4042823e is 50, key is test_row_0/B:col10/1731477037277/Put/seqid=0 2024-11-13T05:50:37,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741907_1083 (size=12301) 2024-11-13T05:50:37,518 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/b7a167c71c134246924a032cfd66e3f9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/b7a167c71c134246924a032cfd66e3f9 2024-11-13T05:50:37,528 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into b7a167c71c134246924a032cfd66e3f9(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:37,528 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:37,528 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=13, startTime=1731477036824; duration=0sec 2024-11-13T05:50:37,528 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:37,528 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:37,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477097536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477097562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477097572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477097572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477097572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477097651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/d03335862ef04362a035f2eb4042823e 2024-11-13T05:50:37,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477097861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:37,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/40ee76ee023744b58caa6f8a48e24e68 is 50, key is test_row_0/C:col10/1731477037277/Put/seqid=0 2024-11-13T05:50:37,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741908_1084 (size=12301) 2024-11-13T05:50:37,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/40ee76ee023744b58caa6f8a48e24e68 2024-11-13T05:50:37,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/3c76b052ae9b4de28248cad6a52c7070 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/3c76b052ae9b4de28248cad6a52c7070 2024-11-13T05:50:37,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/3c76b052ae9b4de28248cad6a52c7070, entries=200, sequenceid=345, filesize=14.4 K 2024-11-13T05:50:37,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/d03335862ef04362a035f2eb4042823e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d03335862ef04362a035f2eb4042823e 2024-11-13T05:50:37,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d03335862ef04362a035f2eb4042823e, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:50:37,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/40ee76ee023744b58caa6f8a48e24e68 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/40ee76ee023744b58caa6f8a48e24e68 2024-11-13T05:50:37,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/40ee76ee023744b58caa6f8a48e24e68, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:50:37,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 9480a359b149cb565364fd0f1f82db91 in 582ms, sequenceid=345, compaction requested=true 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:37,914 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:50:37,915 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:37,917 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:37,917 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:37,917 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:37,917 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/8fba0cb100db42579e32f1a923b247dd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/62bd712a5e4443b99ab327b51e1ad2c0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/3c76b052ae9b4de28248cad6a52c7070] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=39.1 K 2024-11-13T05:50:37,918 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:37,918 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:37,918 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:37,918 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2baf06d5a9a549e0acb05b260a56db13, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/164d76250bc44e75b8c9f17e8d62208b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d03335862ef04362a035f2eb4042823e] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=36.7 K 2024-11-13T05:50:37,919 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2baf06d5a9a549e0acb05b260a56db13, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731477035789 2024-11-13T05:50:37,919 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fba0cb100db42579e32f1a923b247dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731477035789 2024-11-13T05:50:37,919 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 164d76250bc44e75b8c9f17e8d62208b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731477036184 2024-11-13T05:50:37,920 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 62bd712a5e4443b99ab327b51e1ad2c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731477036184 2024-11-13T05:50:37,920 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c76b052ae9b4de28248cad6a52c7070, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477036878 2024-11-13T05:50:37,920 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03335862ef04362a035f2eb4042823e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477036878 2024-11-13T05:50:37,932 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:37,932 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/d3d61daad8c445f8b2eedf628f860142 is 50, key is test_row_0/A:col10/1731477037277/Put/seqid=0 2024-11-13T05:50:37,937 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:37,937 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/5de63ae2139349509734bd9d00a2d30b is 50, key is test_row_0/B:col10/1731477037277/Put/seqid=0 2024-11-13T05:50:37,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741909_1085 (size=13119) 2024-11-13T05:50:37,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741910_1086 (size=13119) 2024-11-13T05:50:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:38,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:50:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:38,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/31f184006b1f46c1b545b968b761508c is 50, key is test_row_0/A:col10/1731477038073/Put/seqid=0 2024-11-13T05:50:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477098101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477098102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477098103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477098106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741911_1087 (size=14741) 2024-11-13T05:50:38,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/31f184006b1f46c1b545b968b761508c 2024-11-13T05:50:38,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/3f3247791bd74998b7799bd31a0e4f57 is 50, key is test_row_0/B:col10/1731477038073/Put/seqid=0 2024-11-13T05:50:38,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477098171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T05:50:38,182 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:38,182 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-13T05:50:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:50:38,186 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:38,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:38,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:38,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741912_1088 (size=12301) 2024-11-13T05:50:38,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/3f3247791bd74998b7799bd31a0e4f57 2024-11-13T05:50:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477098222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477098222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477098222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/27eb5010c28e4ff0b2a488e4dc1aaae3 is 50, key is test_row_0/C:col10/1731477038073/Put/seqid=0 2024-11-13T05:50:38,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477098232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741913_1089 (size=12301) 2024-11-13T05:50:38,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/27eb5010c28e4ff0b2a488e4dc1aaae3 2024-11-13T05:50:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/31f184006b1f46c1b545b968b761508c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/31f184006b1f46c1b545b968b761508c 2024-11-13T05:50:38,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/31f184006b1f46c1b545b968b761508c, entries=200, sequenceid=363, filesize=14.4 K 2024-11-13T05:50:38,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/3f3247791bd74998b7799bd31a0e4f57 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3f3247791bd74998b7799bd31a0e4f57 2024-11-13T05:50:38,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:50:38,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3f3247791bd74998b7799bd31a0e4f57, entries=150, sequenceid=363, filesize=12.0 K 2024-11-13T05:50:38,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/27eb5010c28e4ff0b2a488e4dc1aaae3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/27eb5010c28e4ff0b2a488e4dc1aaae3 2024-11-13T05:50:38,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/27eb5010c28e4ff0b2a488e4dc1aaae3, entries=150, sequenceid=363, filesize=12.0 K 2024-11-13T05:50:38,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9480a359b149cb565364fd0f1f82db91 in 276ms, sequenceid=363, compaction requested=true 2024-11-13T05:50:38,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:50:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T05:50:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:38,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T05:50:38,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T05:50:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:38,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:50:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:38,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:38,364 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/d3d61daad8c445f8b2eedf628f860142 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d3d61daad8c445f8b2eedf628f860142 2024-11-13T05:50:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1085a899e754455cba0359a63353b7f6 is 50, key is test_row_0/A:col10/1731477038100/Put/seqid=0 2024-11-13T05:50:38,373 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/5de63ae2139349509734bd9d00a2d30b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5de63ae2139349509734bd9d00a2d30b 2024-11-13T05:50:38,387 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into d3d61daad8c445f8b2eedf628f860142(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:38,387 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 5de63ae2139349509734bd9d00a2d30b(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:38,387 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:38,387 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:38,387 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=13, startTime=1731477037914; duration=0sec 2024-11-13T05:50:38,387 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=13, startTime=1731477037914; duration=0sec 2024-11-13T05:50:38,387 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T05:50:38,387 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:38,387 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-13T05:50:38,388 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:38,388 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:38,388 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:38,388 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:38,390 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:38,390 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:38,390 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-13T05:50:38,390 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:38,390 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T05:50:38,391 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T05:50:38,391 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. because compaction request was cancelled 2024-11-13T05:50:38,391 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/b7a167c71c134246924a032cfd66e3f9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/4eaf8f3c962348cf88ab1f87eca25c81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/40ee76ee023744b58caa6f8a48e24e68, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/27eb5010c28e4ff0b2a488e4dc1aaae3] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=48.8 K 2024-11-13T05:50:38,391 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:38,392 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a167c71c134246924a032cfd66e3f9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731477035789 2024-11-13T05:50:38,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741914_1090 (size=12301) 2024-11-13T05:50:38,393 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eaf8f3c962348cf88ab1f87eca25c81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731477036184 2024-11-13T05:50:38,394 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ee76ee023744b58caa6f8a48e24e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477036878 2024-11-13T05:50:38,395 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1085a899e754455cba0359a63353b7f6 2024-11-13T05:50:38,395 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T05:50:38,396 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 27eb5010c28e4ff0b2a488e4dc1aaae3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731477037504 2024-11-13T05:50:38,397 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T05:50:38,397 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T05:50:38,397 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. because compaction request was cancelled 2024-11-13T05:50:38,397 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:38,397 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T05:50:38,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T05:50:38,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T05:50:38,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. because compaction request was cancelled 2024-11-13T05:50:38,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:38,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/8d8b27b0959146aa8d31b81378b13b6b is 50, key is test_row_0/B:col10/1731477038100/Put/seqid=0 2024-11-13T05:50:38,414 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:38,415 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fdd3b2e554424fb9b2ae6cb545ea0c1c is 50, key is test_row_0/C:col10/1731477038073/Put/seqid=0 2024-11-13T05:50:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:38,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:38,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741915_1091 (size=12301) 2024-11-13T05:50:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741916_1092 (size=13153) 2024-11-13T05:50:38,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477098473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477098476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477098479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477098493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:50:38,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477098592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477098592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477098593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477098602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477098682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477098801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477098801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:50:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477098812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477098821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:38,851 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/8d8b27b0959146aa8d31b81378b13b6b 2024-11-13T05:50:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/06f04e5879c841c5a9ab749ef266872a is 50, key is test_row_0/C:col10/1731477038100/Put/seqid=0 2024-11-13T05:50:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741917_1093 (size=12301) 2024-11-13T05:50:38,885 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fdd3b2e554424fb9b2ae6cb545ea0c1c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fdd3b2e554424fb9b2ae6cb545ea0c1c 2024-11-13T05:50:38,900 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into fdd3b2e554424fb9b2ae6cb545ea0c1c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:38,900 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:38,900 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=12, startTime=1731477038351; duration=0sec 2024-11-13T05:50:38,901 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:38,901 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477099113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477099113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477099123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477099131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,266 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/06f04e5879c841c5a9ab749ef266872a 2024-11-13T05:50:39,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/1085a899e754455cba0359a63353b7f6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1085a899e754455cba0359a63353b7f6 2024-11-13T05:50:39,283 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1085a899e754455cba0359a63353b7f6, entries=150, sequenceid=378, filesize=12.0 K 2024-11-13T05:50:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/8d8b27b0959146aa8d31b81378b13b6b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/8d8b27b0959146aa8d31b81378b13b6b 2024-11-13T05:50:39,292 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/8d8b27b0959146aa8d31b81378b13b6b, entries=150, sequenceid=378, filesize=12.0 K 2024-11-13T05:50:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/06f04e5879c841c5a9ab749ef266872a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/06f04e5879c841c5a9ab749ef266872a 2024-11-13T05:50:39,305 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/06f04e5879c841c5a9ab749ef266872a, entries=150, sequenceid=378, filesize=12.0 K 2024-11-13T05:50:39,307 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9480a359b149cb565364fd0f1f82db91 in 953ms, sequenceid=378, compaction requested=true 2024-11-13T05:50:39,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:39,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:39,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-13T05:50:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-13T05:50:39,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T05:50:39,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1210 sec 2024-11-13T05:50:39,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1350 sec 2024-11-13T05:50:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T05:50:39,321 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:39,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-13T05:50:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T05:50:39,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:39,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:39,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T05:50:39,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T05:50:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:39,479 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:50:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:39,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/411da41fcad54ddc8f7a7e0b77522a9c is 50, key is test_row_0/A:col10/1731477038464/Put/seqid=0 2024-11-13T05:50:39,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741918_1094 (size=12301) 2024-11-13T05:50:39,534 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/411da41fcad54ddc8f7a7e0b77522a9c 2024-11-13T05:50:39,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/6d895d6f703a415280e9e9ae97db013d is 50, key is test_row_0/B:col10/1731477038464/Put/seqid=0 2024-11-13T05:50:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741919_1095 (size=12301) 2024-11-13T05:50:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:39,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T05:50:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477099687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477099687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477099691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477099695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477099702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,716 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 9480a359b149cb565364fd0f1f82db91 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477099802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477099802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477099802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477099811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T05:50:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/6d895d6f703a415280e9e9ae97db013d 2024-11-13T05:50:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477100011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477100011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477100012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fa856e55a4a541d989d6ecb6bf3f1fc5 is 50, key is test_row_0/C:col10/1731477038464/Put/seqid=0 2024-11-13T05:50:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477100021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741920_1096 (size=12301) 2024-11-13T05:50:40,042 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fa856e55a4a541d989d6ecb6bf3f1fc5 2024-11-13T05:50:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/411da41fcad54ddc8f7a7e0b77522a9c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/411da41fcad54ddc8f7a7e0b77522a9c 2024-11-13T05:50:40,062 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/411da41fcad54ddc8f7a7e0b77522a9c, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:50:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/6d895d6f703a415280e9e9ae97db013d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/6d895d6f703a415280e9e9ae97db013d 2024-11-13T05:50:40,085 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/6d895d6f703a415280e9e9ae97db013d, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:50:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/fa856e55a4a541d989d6ecb6bf3f1fc5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fa856e55a4a541d989d6ecb6bf3f1fc5 2024-11-13T05:50:40,099 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fa856e55a4a541d989d6ecb6bf3f1fc5, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:50:40,101 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9480a359b149cb565364fd0f1f82db91 in 622ms, sequenceid=398, compaction requested=true 2024-11-13T05:50:40,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-13T05:50:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-13T05:50:40,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T05:50:40,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 779 msec 2024-11-13T05:50:40,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 785 msec 2024-11-13T05:50:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:40,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:50:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:40,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/ac2edec195d4487a9693f423565e5ce8 is 50, key is test_row_0/A:col10/1731477039645/Put/seqid=0 2024-11-13T05:50:40,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741921_1097 (size=14741) 2024-11-13T05:50:40,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/ac2edec195d4487a9693f423565e5ce8 2024-11-13T05:50:40,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/06921fb122214489b31bd19c43ccdeb2 is 50, key is test_row_0/B:col10/1731477039645/Put/seqid=0 2024-11-13T05:50:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477100349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477100357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741922_1098 (size=12301) 2024-11-13T05:50:40,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/06921fb122214489b31bd19c43ccdeb2 2024-11-13T05:50:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477100358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477100360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d4b92b96551e48178fdaa0a83783e60c is 50, key is test_row_0/C:col10/1731477039645/Put/seqid=0 2024-11-13T05:50:40,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741923_1099 (size=12301) 2024-11-13T05:50:40,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d4b92b96551e48178fdaa0a83783e60c 2024-11-13T05:50:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/ac2edec195d4487a9693f423565e5ce8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/ac2edec195d4487a9693f423565e5ce8 2024-11-13T05:50:40,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/ac2edec195d4487a9693f423565e5ce8, entries=200, sequenceid=414, filesize=14.4 K 2024-11-13T05:50:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/06921fb122214489b31bd19c43ccdeb2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/06921fb122214489b31bd19c43ccdeb2 2024-11-13T05:50:40,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/06921fb122214489b31bd19c43ccdeb2, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T05:50:40,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d4b92b96551e48178fdaa0a83783e60c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d4b92b96551e48178fdaa0a83783e60c 2024-11-13T05:50:40,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d4b92b96551e48178fdaa0a83783e60c, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T05:50:40,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9480a359b149cb565364fd0f1f82db91 in 134ms, sequenceid=414, compaction requested=true 2024-11-13T05:50:40,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:40,458 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:50:40,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:40,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:40,458 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:50:40,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:40,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:40,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:40,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T05:50:40,461 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:40,462 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:40,463 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67203 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:50:40,463 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:40,463 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,463 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d3d61daad8c445f8b2eedf628f860142, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/31f184006b1f46c1b545b968b761508c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1085a899e754455cba0359a63353b7f6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/411da41fcad54ddc8f7a7e0b77522a9c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/ac2edec195d4487a9693f423565e5ce8] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=65.6 K 2024-11-13T05:50:40,464 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62323 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:50:40,464 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:40,464 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,464 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5de63ae2139349509734bd9d00a2d30b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3f3247791bd74998b7799bd31a0e4f57, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/8d8b27b0959146aa8d31b81378b13b6b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/6d895d6f703a415280e9e9ae97db013d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/06921fb122214489b31bd19c43ccdeb2] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=60.9 K 2024-11-13T05:50:40,466 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de63ae2139349509734bd9d00a2d30b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477036878 2024-11-13T05:50:40,466 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d61daad8c445f8b2eedf628f860142, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477036878 2024-11-13T05:50:40,467 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f3247791bd74998b7799bd31a0e4f57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731477037504 2024-11-13T05:50:40,467 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31f184006b1f46c1b545b968b761508c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731477037504 2024-11-13T05:50:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-13T05:50:40,469 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8b27b0959146aa8d31b81378b13b6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731477038100 2024-11-13T05:50:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:50:40,470 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1085a899e754455cba0359a63353b7f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731477038100 2024-11-13T05:50:40,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:40,471 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d895d6f703a415280e9e9ae97db013d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731477038464 2024-11-13T05:50:40,471 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 411da41fcad54ddc8f7a7e0b77522a9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731477038464 2024-11-13T05:50:40,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:40,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:40,473 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac2edec195d4487a9693f423565e5ce8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731477039645 2024-11-13T05:50:40,474 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 06921fb122214489b31bd19c43ccdeb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731477039645 2024-11-13T05:50:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:40,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:50:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:40,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:40,496 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#87 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:40,496 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/95782a33db2148d6a52360c649385f16 is 50, key is test_row_0/B:col10/1731477039645/Put/seqid=0 2024-11-13T05:50:40,507 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:40,508 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/df5eda60733e4ac9a7f454a3d4bb8eb8 is 50, key is test_row_0/A:col10/1731477039645/Put/seqid=0 2024-11-13T05:50:40,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/913129d7d7f748278f67a808005c24ba is 50, key is test_row_0/A:col10/1731477040479/Put/seqid=0 2024-11-13T05:50:40,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477100503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477100506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477100512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477100512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741924_1100 (size=13289) 2024-11-13T05:50:40,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741926_1102 (size=12301) 2024-11-13T05:50:40,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741925_1101 (size=13289) 2024-11-13T05:50:40,541 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/df5eda60733e4ac9a7f454a3d4bb8eb8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/df5eda60733e4ac9a7f454a3d4bb8eb8 2024-11-13T05:50:40,548 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into df5eda60733e4ac9a7f454a3d4bb8eb8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:40,549 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:40,549 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=11, startTime=1731477040458; duration=0sec 2024-11-13T05:50:40,549 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:40,549 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:40,549 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:40,555 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:40,555 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:40,555 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,555 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fdd3b2e554424fb9b2ae6cb545ea0c1c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/06f04e5879c841c5a9ab749ef266872a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fa856e55a4a541d989d6ecb6bf3f1fc5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d4b92b96551e48178fdaa0a83783e60c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=48.9 K 2024-11-13T05:50:40,557 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdd3b2e554424fb9b2ae6cb545ea0c1c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731477037504 2024-11-13T05:50:40,557 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f04e5879c841c5a9ab749ef266872a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731477038100 2024-11-13T05:50:40,558 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa856e55a4a541d989d6ecb6bf3f1fc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731477038464 2024-11-13T05:50:40,558 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b92b96551e48178fdaa0a83783e60c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731477039645 2024-11-13T05:50:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:50:40,579 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:40,580 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/07f49fe691b74a6da438037d6ed6d7af is 50, key is test_row_0/C:col10/1731477039645/Put/seqid=0 2024-11-13T05:50:40,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741927_1103 (size=13289) 2024-11-13T05:50:40,607 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/07f49fe691b74a6da438037d6ed6d7af as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/07f49fe691b74a6da438037d6ed6d7af 2024-11-13T05:50:40,625 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into 07f49fe691b74a6da438037d6ed6d7af(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:40,625 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:40,625 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=12, startTime=1731477040459; duration=0sec 2024-11-13T05:50:40,625 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:40,625 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477100622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477100622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477100623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T05:50:40,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477100623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:50:40,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T05:50:40,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:40,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477100832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477100832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477100842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477100851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:40,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/913129d7d7f748278f67a808005c24ba 2024-11-13T05:50:40,935 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/95782a33db2148d6a52360c649385f16 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/95782a33db2148d6a52360c649385f16 2024-11-13T05:50:40,947 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 95782a33db2148d6a52360c649385f16(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:40,947 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:40,947 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=11, startTime=1731477040458; duration=0sec 2024-11-13T05:50:40,947 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:40,947 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:40,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T05:50:40,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/b08f1cb9960542189af344da67e412d5 is 50, key is test_row_0/B:col10/1731477040479/Put/seqid=0 2024-11-13T05:50:40,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:40,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:40,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741928_1104 (size=12301) 2024-11-13T05:50:40,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/b08f1cb9960542189af344da67e412d5 2024-11-13T05:50:41,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1c61729146c94b0da01e9096dbec56b8 is 50, key is test_row_0/C:col10/1731477040479/Put/seqid=0 2024-11-13T05:50:41,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741929_1105 (size=12301) 2024-11-13T05:50:41,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1c61729146c94b0da01e9096dbec56b8 2024-11-13T05:50:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/913129d7d7f748278f67a808005c24ba as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/913129d7d7f748278f67a808005c24ba 2024-11-13T05:50:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:50:41,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/913129d7d7f748278f67a808005c24ba, entries=150, sequenceid=434, filesize=12.0 K 2024-11-13T05:50:41,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/b08f1cb9960542189af344da67e412d5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/b08f1cb9960542189af344da67e412d5 2024-11-13T05:50:41,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/b08f1cb9960542189af344da67e412d5, entries=150, sequenceid=434, filesize=12.0 K 2024-11-13T05:50:41,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/1c61729146c94b0da01e9096dbec56b8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1c61729146c94b0da01e9096dbec56b8 2024-11-13T05:50:41,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T05:50:41,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:41,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:41,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:41,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1c61729146c94b0da01e9096dbec56b8, entries=150, sequenceid=434, filesize=12.0 K 2024-11-13T05:50:41,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9480a359b149cb565364fd0f1f82db91 in 637ms, sequenceid=434, compaction requested=false 2024-11-13T05:50:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:41,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:50:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:41,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fda85376274847f6bd04036bc4fa464b is 50, key is test_row_0/A:col10/1731477040489/Put/seqid=0 2024-11-13T05:50:41,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741930_1106 (size=12301) 2024-11-13T05:50:41,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477101185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fda85376274847f6bd04036bc4fa464b 2024-11-13T05:50:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477101193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477101197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477101197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/5f1b3aad8d8643d9911e533cdf5915c9 is 50, key is test_row_0/B:col10/1731477040489/Put/seqid=0 2024-11-13T05:50:41,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741931_1107 (size=12301) 2024-11-13T05:50:41,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/5f1b3aad8d8643d9911e533cdf5915c9 2024-11-13T05:50:41,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6ad42e76dedd4e41a0db3ca4f81bb0ed is 50, key is test_row_0/C:col10/1731477040489/Put/seqid=0 2024-11-13T05:50:41,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741932_1108 (size=12301) 2024-11-13T05:50:41,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6ad42e76dedd4e41a0db3ca4f81bb0ed 2024-11-13T05:50:41,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T05:50:41,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fda85376274847f6bd04036bc4fa464b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fda85376274847f6bd04036bc4fa464b 2024-11-13T05:50:41,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fda85376274847f6bd04036bc4fa464b, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T05:50:41,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/5f1b3aad8d8643d9911e533cdf5915c9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5f1b3aad8d8643d9911e533cdf5915c9 2024-11-13T05:50:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:41,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:41,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:41,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5f1b3aad8d8643d9911e533cdf5915c9, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T05:50:41,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6ad42e76dedd4e41a0db3ca4f81bb0ed as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6ad42e76dedd4e41a0db3ca4f81bb0ed 2024-11-13T05:50:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477101301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477101312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477101312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477101313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6ad42e76dedd4e41a0db3ca4f81bb0ed, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T05:50:41,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9480a359b149cb565364fd0f1f82db91 in 180ms, sequenceid=450, compaction requested=true 2024-11-13T05:50:41,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:41,326 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:41,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:41,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:41,326 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:41,328 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:41,328 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:41,328 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,328 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/95782a33db2148d6a52360c649385f16, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/b08f1cb9960542189af344da67e412d5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5f1b3aad8d8643d9911e533cdf5915c9] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=37.0 K 2024-11-13T05:50:41,329 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 95782a33db2148d6a52360c649385f16, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731477039645 2024-11-13T05:50:41,331 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:41,331 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b08f1cb9960542189af344da67e412d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731477040357 2024-11-13T05:50:41,331 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:41,331 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,331 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/df5eda60733e4ac9a7f454a3d4bb8eb8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/913129d7d7f748278f67a808005c24ba, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fda85376274847f6bd04036bc4fa464b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=37.0 K 2024-11-13T05:50:41,332 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5eda60733e4ac9a7f454a3d4bb8eb8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731477039645 2024-11-13T05:50:41,332 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1b3aad8d8643d9911e533cdf5915c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477040489 2024-11-13T05:50:41,332 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913129d7d7f748278f67a808005c24ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731477040357 2024-11-13T05:50:41,333 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda85376274847f6bd04036bc4fa464b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477040489 2024-11-13T05:50:41,343 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:41,344 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/3c45bb91f26a49b0898f987d0ae71ef6 is 50, key is test_row_0/B:col10/1731477040489/Put/seqid=0 2024-11-13T05:50:41,353 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:41,354 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/a1966be9705e4891b6b91fd264c37a0f is 50, key is test_row_0/A:col10/1731477040489/Put/seqid=0 2024-11-13T05:50:41,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741933_1109 (size=13391) 2024-11-13T05:50:41,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741934_1110 (size=13391) 2024-11-13T05:50:41,388 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/a1966be9705e4891b6b91fd264c37a0f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a1966be9705e4891b6b91fd264c37a0f 2024-11-13T05:50:41,398 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into a1966be9705e4891b6b91fd264c37a0f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:41,398 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:41,398 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=13, startTime=1731477041326; duration=0sec 2024-11-13T05:50:41,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:41,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:41,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:41,403 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:41,403 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:41,403 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,404 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/07f49fe691b74a6da438037d6ed6d7af, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1c61729146c94b0da01e9096dbec56b8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6ad42e76dedd4e41a0db3ca4f81bb0ed] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=37.0 K 2024-11-13T05:50:41,404 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f49fe691b74a6da438037d6ed6d7af, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731477039645 2024-11-13T05:50:41,405 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c61729146c94b0da01e9096dbec56b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731477040357 2024-11-13T05:50:41,405 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad42e76dedd4e41a0db3ca4f81bb0ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477040489 2024-11-13T05:50:41,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T05:50:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:41,448 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T05:50:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:41,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:41,454 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:41,459 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/cd64a140245d4026a44053264be5503a is 50, key is test_row_0/C:col10/1731477040489/Put/seqid=0 2024-11-13T05:50:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fe0ec34f8e1745728d8bf9a3c363f136 is 50, key is test_row_0/A:col10/1731477041182/Put/seqid=0 2024-11-13T05:50:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:41,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:41,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741935_1111 (size=13391) 2024-11-13T05:50:41,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741936_1112 (size=12301) 2024-11-13T05:50:41,528 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fe0ec34f8e1745728d8bf9a3c363f136 2024-11-13T05:50:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/4177e5701beb4bd5bafea4e4c39ef789 is 50, key is test_row_0/B:col10/1731477041182/Put/seqid=0 2024-11-13T05:50:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477101549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477101553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477101561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477101563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741937_1113 (size=12301) 2024-11-13T05:50:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:50:41,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477101672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477101672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477101672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477101681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52320 deadline: 1731477101741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,744 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 9480a359b149cb565364fd0f1f82db91 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,769 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/3c45bb91f26a49b0898f987d0ae71ef6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3c45bb91f26a49b0898f987d0ae71ef6 2024-11-13T05:50:41,774 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into 3c45bb91f26a49b0898f987d0ae71ef6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:41,774 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:41,774 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=13, startTime=1731477041326; duration=0sec 2024-11-13T05:50:41,774 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:41,774 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:41,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477101881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477101891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477101892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477101892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:41,929 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/cd64a140245d4026a44053264be5503a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/cd64a140245d4026a44053264be5503a 2024-11-13T05:50:41,939 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into cd64a140245d4026a44053264be5503a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:41,939 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:41,939 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=13, startTime=1731477041327; duration=0sec 2024-11-13T05:50:41,939 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:41,939 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:41,983 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/4177e5701beb4bd5bafea4e4c39ef789 2024-11-13T05:50:42,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6222a0962cc44dacb1ffe92acc3ee0d5 is 50, key is test_row_0/C:col10/1731477041182/Put/seqid=0 2024-11-13T05:50:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741938_1114 (size=12301) 2024-11-13T05:50:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477102192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477102201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477102201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477102202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,441 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6222a0962cc44dacb1ffe92acc3ee0d5 2024-11-13T05:50:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fe0ec34f8e1745728d8bf9a3c363f136 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fe0ec34f8e1745728d8bf9a3c363f136 2024-11-13T05:50:42,453 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fe0ec34f8e1745728d8bf9a3c363f136, entries=150, sequenceid=470, filesize=12.0 K 2024-11-13T05:50:42,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/4177e5701beb4bd5bafea4e4c39ef789 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4177e5701beb4bd5bafea4e4c39ef789 2024-11-13T05:50:42,479 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4177e5701beb4bd5bafea4e4c39ef789, entries=150, sequenceid=470, filesize=12.0 K 2024-11-13T05:50:42,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/6222a0962cc44dacb1ffe92acc3ee0d5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6222a0962cc44dacb1ffe92acc3ee0d5 2024-11-13T05:50:42,486 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6222a0962cc44dacb1ffe92acc3ee0d5, entries=150, sequenceid=470, filesize=12.0 K 2024-11-13T05:50:42,489 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 9480a359b149cb565364fd0f1f82db91 in 1040ms, sequenceid=470, compaction requested=false 2024-11-13T05:50:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-13T05:50:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-13T05:50:42,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-13T05:50:42,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-13T05:50:42,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.0380 sec 2024-11-13T05:50:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T05:50:42,612 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:42,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-13T05:50:42,614 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:42,615 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:42,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T05:50:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:42,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:50:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:42,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T05:50:42,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/abadf747bb174366832090d811e882bb is 50, key is test_row_0/A:col10/1731477041545/Put/seqid=0 2024-11-13T05:50:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741939_1115 (size=14741) 2024-11-13T05:50:42,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/abadf747bb174366832090d811e882bb 2024-11-13T05:50:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477102738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477102738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477102739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477102739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/fd5f78d2d91d467a84b55743c1c2f78f is 50, key is test_row_0/B:col10/1731477041545/Put/seqid=0 2024-11-13T05:50:42,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:42,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:42,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:42,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:42,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:42,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:42,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741940_1116 (size=12301) 2024-11-13T05:50:42,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:42,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:42,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T05:50:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477103062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477103063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477103063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477103063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/fd5f78d2d91d467a84b55743c1c2f78f 2024-11-13T05:50:43,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d658aedf73784f4c95b1bdee63036ebe is 50, key is test_row_0/C:col10/1731477041545/Put/seqid=0 2024-11-13T05:50:43,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741941_1117 (size=12301) 2024-11-13T05:50:43,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T05:50:43,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52294 deadline: 1731477103382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52330 deadline: 1731477103382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52360 deadline: 1731477103382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1731477103382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:43,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:50:43,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:43,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:43,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:43,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d658aedf73784f4c95b1bdee63036ebe 2024-11-13T05:50:43,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/abadf747bb174366832090d811e882bb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/abadf747bb174366832090d811e882bb 2024-11-13T05:50:43,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/abadf747bb174366832090d811e882bb, entries=200, sequenceid=487, filesize=14.4 K 2024-11-13T05:50:43,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/fd5f78d2d91d467a84b55743c1c2f78f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/fd5f78d2d91d467a84b55743c1c2f78f 2024-11-13T05:50:43,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/fd5f78d2d91d467a84b55743c1c2f78f, entries=150, sequenceid=487, filesize=12.0 K 2024-11-13T05:50:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/d658aedf73784f4c95b1bdee63036ebe as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d658aedf73784f4c95b1bdee63036ebe 2024-11-13T05:50:43,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d658aedf73784f4c95b1bdee63036ebe, entries=150, sequenceid=487, filesize=12.0 K 2024-11-13T05:50:43,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9480a359b149cb565364fd0f1f82db91 in 978ms, sequenceid=487, compaction requested=true 2024-11-13T05:50:43,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:43,684 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:43,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:43,684 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:43,685 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:43,685 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/A is initiating minor compaction (all files) 2024-11-13T05:50:43,686 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/A in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,686 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a1966be9705e4891b6b91fd264c37a0f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fe0ec34f8e1745728d8bf9a3c363f136, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/abadf747bb174366832090d811e882bb] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=39.5 K 2024-11-13T05:50:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9480a359b149cb565364fd0f1f82db91:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:43,686 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1966be9705e4891b6b91fd264c37a0f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477040489 2024-11-13T05:50:43,687 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:43,687 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/B is initiating minor compaction (all files) 2024-11-13T05:50:43,687 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/B in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,687 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3c45bb91f26a49b0898f987d0ae71ef6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4177e5701beb4bd5bafea4e4c39ef789, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/fd5f78d2d91d467a84b55743c1c2f78f] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=37.1 K 2024-11-13T05:50:43,687 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe0ec34f8e1745728d8bf9a3c363f136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731477041178 2024-11-13T05:50:43,689 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting abadf747bb174366832090d811e882bb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731477041545 2024-11-13T05:50:43,689 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c45bb91f26a49b0898f987d0ae71ef6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477040489 2024-11-13T05:50:43,690 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4177e5701beb4bd5bafea4e4c39ef789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731477041178 2024-11-13T05:50:43,690 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5f78d2d91d467a84b55743c1c2f78f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731477041545 2024-11-13T05:50:43,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T05:50:43,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,701 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:50:43,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:43,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:43,713 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#A#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:43,713 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9e435d19f6ff4b0f9d8a83c08e16dafd is 50, key is test_row_0/A:col10/1731477041545/Put/seqid=0 2024-11-13T05:50:43,725 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:43,725 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ad7da4d975824e7480f95dec510b2f4c is 50, key is test_row_0/B:col10/1731477041545/Put/seqid=0 2024-11-13T05:50:43,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fbe0a81e7f0844c594671a0f0712b6f3 is 50, key is test_row_0/A:col10/1731477042736/Put/seqid=0 2024-11-13T05:50:43,739 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T05:50:43,739 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741942_1118 (size=13493) 2024-11-13T05:50:43,741 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T05:50:43,741 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,742 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T05:50:43,742 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741943_1119 (size=13493) 2024-11-13T05:50:43,744 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T05:50:43,744 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741944_1120 (size=12301) 2024-11-13T05:50:43,748 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/ad7da4d975824e7480f95dec510b2f4c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7da4d975824e7480f95dec510b2f4c 2024-11-13T05:50:43,749 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,750 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T05:50:43,755 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/B of 9480a359b149cb565364fd0f1f82db91 into ad7da4d975824e7480f95dec510b2f4c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:43,755 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:43,756 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/B, priority=13, startTime=1731477043684; duration=0sec 2024-11-13T05:50:43,756 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:43,756 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:B 2024-11-13T05:50:43,756 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:50:43,757 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:50:43,757 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 9480a359b149cb565364fd0f1f82db91/C is initiating minor compaction (all files) 2024-11-13T05:50:43,757 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9480a359b149cb565364fd0f1f82db91/C in TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:43,757 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/cd64a140245d4026a44053264be5503a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6222a0962cc44dacb1ffe92acc3ee0d5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d658aedf73784f4c95b1bdee63036ebe] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp, totalSize=37.1 K 2024-11-13T05:50:43,758 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting cd64a140245d4026a44053264be5503a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477040489 2024-11-13T05:50:43,758 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6222a0962cc44dacb1ffe92acc3ee0d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731477041178 2024-11-13T05:50:43,759 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d658aedf73784f4c95b1bdee63036ebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731477041545 2024-11-13T05:50:43,765 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9480a359b149cb565364fd0f1f82db91#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:43,765 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/84410c48565c48e9930e2dde6a19914a is 50, key is test_row_0/C:col10/1731477041545/Put/seqid=0 2024-11-13T05:50:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741945_1121 (size=13493) 2024-11-13T05:50:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:43,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. as already flushing 2024-11-13T05:50:43,895 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T05:50:43,896 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T05:50:43,896 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T05:50:43,896 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,896 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,896 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,896 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,896 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,896 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,896 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,896 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,896 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:43,898 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T05:50:43,898 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:43,898 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,898 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:43,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:44,147 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/9e435d19f6ff4b0f9d8a83c08e16dafd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9e435d19f6ff4b0f9d8a83c08e16dafd 2024-11-13T05:50:44,149 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fbe0a81e7f0844c594671a0f0712b6f3 2024-11-13T05:50:44,155 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/A of 9480a359b149cb565364fd0f1f82db91 into 9e435d19f6ff4b0f9d8a83c08e16dafd(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:44,156 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:44,156 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/A, priority=13, startTime=1731477043683; duration=0sec 2024-11-13T05:50:44,156 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:44,156 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:A 2024-11-13T05:50:44,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/2c7864cc3b0345dabb5bb7c85c159cfb is 50, key is test_row_0/B:col10/1731477042736/Put/seqid=0 2024-11-13T05:50:44,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741946_1122 (size=12301) 2024-11-13T05:50:44,167 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/2c7864cc3b0345dabb5bb7c85c159cfb 2024-11-13T05:50:44,178 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/84410c48565c48e9930e2dde6a19914a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/84410c48565c48e9930e2dde6a19914a 2024-11-13T05:50:44,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/064b303bb81c4517953d489d735d8455 is 50, key is test_row_0/C:col10/1731477042736/Put/seqid=0 2024-11-13T05:50:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741947_1123 (size=12301) 2024-11-13T05:50:44,185 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/064b303bb81c4517953d489d735d8455 2024-11-13T05:50:44,185 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9480a359b149cb565364fd0f1f82db91/C of 9480a359b149cb565364fd0f1f82db91 into 84410c48565c48e9930e2dde6a19914a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:44,186 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:44,186 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., storeName=9480a359b149cb565364fd0f1f82db91/C, priority=13, startTime=1731477043685; duration=0sec 2024-11-13T05:50:44,186 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:44,186 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9480a359b149cb565364fd0f1f82db91:C 2024-11-13T05:50:44,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/fbe0a81e7f0844c594671a0f0712b6f3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fbe0a81e7f0844c594671a0f0712b6f3 2024-11-13T05:50:44,197 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fbe0a81e7f0844c594671a0f0712b6f3, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T05:50:44,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/2c7864cc3b0345dabb5bb7c85c159cfb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2c7864cc3b0345dabb5bb7c85c159cfb 2024-11-13T05:50:44,204 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2c7864cc3b0345dabb5bb7c85c159cfb, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T05:50:44,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/064b303bb81c4517953d489d735d8455 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/064b303bb81c4517953d489d735d8455 2024-11-13T05:50:44,210 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/064b303bb81c4517953d489d735d8455, entries=150, sequenceid=504, filesize=12.0 K 2024-11-13T05:50:44,211 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 9480a359b149cb565364fd0f1f82db91 in 510ms, sequenceid=504, compaction requested=false 2024-11-13T05:50:44,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 9480a359b149cb565364fd0f1f82db91: 2024-11-13T05:50:44,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:44,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-13T05:50:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-13T05:50:44,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T05:50:44,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5980 sec 2024-11-13T05:50:44,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.6040 sec 2024-11-13T05:50:44,495 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:50:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T05:50:44,772 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:45,773 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T05:50:45,773 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:50:45,774 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:45,774 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:45,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1911 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1904 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 899 2024-11-13T05:50:45,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2696 rows 2024-11-13T05:50:45,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 895 2024-11-13T05:50:45,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2683 rows 2024-11-13T05:50:45,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:50:45,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:50:45,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:45,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:45,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:50:45,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:50:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:50:45,787 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477045787"}]},"ts":"1731477045787"} 2024-11-13T05:50:45,790 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:50:45,790 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:50:45,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:50:45,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, UNASSIGN}] 2024-11-13T05:50:45,798 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, UNASSIGN 2024-11-13T05:50:45,800 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9480a359b149cb565364fd0f1f82db91, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:45,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, UNASSIGN because future has completed 2024-11-13T05:50:45,802 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:50:45,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:50:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:50:45,962 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:45,962 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:50:45,963 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 9480a359b149cb565364fd0f1f82db91, disabling compactions & flushes 2024-11-13T05:50:45,963 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:45,963 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:45,964 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. after waiting 0 ms 2024-11-13T05:50:45,964 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:45,964 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 9480a359b149cb565364fd0f1f82db91 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T05:50:45,964 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=A 2024-11-13T05:50:45,965 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:45,965 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=B 2024-11-13T05:50:45,965 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:45,965 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9480a359b149cb565364fd0f1f82db91, store=C 2024-11-13T05:50:45,965 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:45,974 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/2d2f5b6bae23456d9905be11074c430f is 50, key is test_row_0/A:col10/1731477045772/Put/seqid=0 2024-11-13T05:50:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741948_1124 (size=9857) 2024-11-13T05:50:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:50:46,380 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/2d2f5b6bae23456d9905be11074c430f 2024-11-13T05:50:46,391 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/84530f0680444f67b0933be12705ae52 is 50, key is test_row_0/B:col10/1731477045772/Put/seqid=0 2024-11-13T05:50:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741949_1125 (size=9857) 2024-11-13T05:50:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:50:46,797 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/84530f0680444f67b0933be12705ae52 2024-11-13T05:50:46,811 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/39f7dfaa18ce499da14558576541b94d is 50, key is test_row_0/C:col10/1731477045772/Put/seqid=0 2024-11-13T05:50:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741950_1126 (size=9857) 2024-11-13T05:50:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:50:47,216 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/39f7dfaa18ce499da14558576541b94d 2024-11-13T05:50:47,222 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/A/2d2f5b6bae23456d9905be11074c430f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/2d2f5b6bae23456d9905be11074c430f 2024-11-13T05:50:47,227 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/2d2f5b6bae23456d9905be11074c430f, entries=100, sequenceid=515, filesize=9.6 K 2024-11-13T05:50:47,228 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/B/84530f0680444f67b0933be12705ae52 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/84530f0680444f67b0933be12705ae52 2024-11-13T05:50:47,232 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/84530f0680444f67b0933be12705ae52, entries=100, sequenceid=515, filesize=9.6 K 2024-11-13T05:50:47,233 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/.tmp/C/39f7dfaa18ce499da14558576541b94d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/39f7dfaa18ce499da14558576541b94d 2024-11-13T05:50:47,236 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/39f7dfaa18ce499da14558576541b94d, entries=100, sequenceid=515, filesize=9.6 K 2024-11-13T05:50:47,237 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9480a359b149cb565364fd0f1f82db91 in 1273ms, sequenceid=515, compaction requested=true 2024-11-13T05:50:47,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/db91e3d831bf4eaf83bda4e137197948, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9b60a6a5e5c4374ac26e6d9c20210f7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e7332592e9ba4bd9b26edce30db2435c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9982fca6c3214783a20bc35eb0c4be21, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d9ae103b09ec42179089bc1b4a3962ae, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/735fcc2a96094f938973992eea520ff3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e631cabbd84a495b930f500baba07da1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1ad92fe2b4d240568fb2580bad240cf1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0e84963a6df24bdb9e22cc2d092b2a69, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/97d5713497ea48c9a2342a55ab9657b4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d48430f766a34fb79eb554cf7590ac14, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9cc2f02b926a4de1901a779f6013546b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/895041719e424a94ae5a7336f1ac410b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1a51ff7f19494beea6f8bfd7839300bf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9335048a5d5452a85ff8961e927e2eb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0a66b6c502c24fb9ac80e9d727015331, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/36ac3bb587994f179e144b3e69d125d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a7b6a48728fe41c5be3caf3422beca56, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/c9df3410c33545919f7d962757a46c54, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/647b087db86349b598a9a5199c8e1618, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/8fba0cb100db42579e32f1a923b247dd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/28c9e47347eb4d708c495a14dcf88acd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/62bd712a5e4443b99ab327b51e1ad2c0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/3c76b052ae9b4de28248cad6a52c7070, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d3d61daad8c445f8b2eedf628f860142, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/31f184006b1f46c1b545b968b761508c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1085a899e754455cba0359a63353b7f6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/411da41fcad54ddc8f7a7e0b77522a9c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/ac2edec195d4487a9693f423565e5ce8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/df5eda60733e4ac9a7f454a3d4bb8eb8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/913129d7d7f748278f67a808005c24ba, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a1966be9705e4891b6b91fd264c37a0f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fda85376274847f6bd04036bc4fa464b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fe0ec34f8e1745728d8bf9a3c363f136, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/abadf747bb174366832090d811e882bb] to archive 2024-11-13T05:50:47,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:50:47,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/db91e3d831bf4eaf83bda4e137197948 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/db91e3d831bf4eaf83bda4e137197948 2024-11-13T05:50:47,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9b60a6a5e5c4374ac26e6d9c20210f7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9b60a6a5e5c4374ac26e6d9c20210f7 2024-11-13T05:50:47,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e7332592e9ba4bd9b26edce30db2435c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e7332592e9ba4bd9b26edce30db2435c 2024-11-13T05:50:47,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9982fca6c3214783a20bc35eb0c4be21 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9982fca6c3214783a20bc35eb0c4be21 2024-11-13T05:50:47,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d9ae103b09ec42179089bc1b4a3962ae to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d9ae103b09ec42179089bc1b4a3962ae 2024-11-13T05:50:47,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/735fcc2a96094f938973992eea520ff3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/735fcc2a96094f938973992eea520ff3 2024-11-13T05:50:47,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e631cabbd84a495b930f500baba07da1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/e631cabbd84a495b930f500baba07da1 2024-11-13T05:50:47,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1ad92fe2b4d240568fb2580bad240cf1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1ad92fe2b4d240568fb2580bad240cf1 2024-11-13T05:50:47,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0e84963a6df24bdb9e22cc2d092b2a69 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0e84963a6df24bdb9e22cc2d092b2a69 2024-11-13T05:50:47,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/97d5713497ea48c9a2342a55ab9657b4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/97d5713497ea48c9a2342a55ab9657b4 2024-11-13T05:50:47,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d48430f766a34fb79eb554cf7590ac14 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d48430f766a34fb79eb554cf7590ac14 2024-11-13T05:50:47,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9cc2f02b926a4de1901a779f6013546b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9cc2f02b926a4de1901a779f6013546b 2024-11-13T05:50:47,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/895041719e424a94ae5a7336f1ac410b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/895041719e424a94ae5a7336f1ac410b 2024-11-13T05:50:47,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1a51ff7f19494beea6f8bfd7839300bf to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1a51ff7f19494beea6f8bfd7839300bf 2024-11-13T05:50:47,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9335048a5d5452a85ff8961e927e2eb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/b9335048a5d5452a85ff8961e927e2eb 2024-11-13T05:50:47,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0a66b6c502c24fb9ac80e9d727015331 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/0a66b6c502c24fb9ac80e9d727015331 2024-11-13T05:50:47,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/36ac3bb587994f179e144b3e69d125d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/36ac3bb587994f179e144b3e69d125d9 2024-11-13T05:50:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a7b6a48728fe41c5be3caf3422beca56 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a7b6a48728fe41c5be3caf3422beca56 2024-11-13T05:50:47,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/c9df3410c33545919f7d962757a46c54 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/c9df3410c33545919f7d962757a46c54 2024-11-13T05:50:47,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/647b087db86349b598a9a5199c8e1618 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/647b087db86349b598a9a5199c8e1618 2024-11-13T05:50:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/8fba0cb100db42579e32f1a923b247dd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/8fba0cb100db42579e32f1a923b247dd 2024-11-13T05:50:47,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/28c9e47347eb4d708c495a14dcf88acd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/28c9e47347eb4d708c495a14dcf88acd 2024-11-13T05:50:47,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/62bd712a5e4443b99ab327b51e1ad2c0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/62bd712a5e4443b99ab327b51e1ad2c0 2024-11-13T05:50:47,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/3c76b052ae9b4de28248cad6a52c7070 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/3c76b052ae9b4de28248cad6a52c7070 2024-11-13T05:50:47,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d3d61daad8c445f8b2eedf628f860142 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/d3d61daad8c445f8b2eedf628f860142 2024-11-13T05:50:47,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/31f184006b1f46c1b545b968b761508c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/31f184006b1f46c1b545b968b761508c 2024-11-13T05:50:47,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1085a899e754455cba0359a63353b7f6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/1085a899e754455cba0359a63353b7f6 2024-11-13T05:50:47,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/411da41fcad54ddc8f7a7e0b77522a9c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/411da41fcad54ddc8f7a7e0b77522a9c 2024-11-13T05:50:47,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/ac2edec195d4487a9693f423565e5ce8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/ac2edec195d4487a9693f423565e5ce8 2024-11-13T05:50:47,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/df5eda60733e4ac9a7f454a3d4bb8eb8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/df5eda60733e4ac9a7f454a3d4bb8eb8 2024-11-13T05:50:47,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/913129d7d7f748278f67a808005c24ba to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/913129d7d7f748278f67a808005c24ba 2024-11-13T05:50:47,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a1966be9705e4891b6b91fd264c37a0f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/a1966be9705e4891b6b91fd264c37a0f 2024-11-13T05:50:47,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fda85376274847f6bd04036bc4fa464b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fda85376274847f6bd04036bc4fa464b 2024-11-13T05:50:47,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fe0ec34f8e1745728d8bf9a3c363f136 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fe0ec34f8e1745728d8bf9a3c363f136 2024-11-13T05:50:47,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/abadf747bb174366832090d811e882bb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/abadf747bb174366832090d811e882bb 2024-11-13T05:50:47,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/de241b95421546b88ee089ed81cf39cb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1a01420dc631470b9926d92ee6c3a26d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d8050035cdbb47eab2026905ab489594, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/277ed23bd572469690923c228eebe18d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/c8828ed585f44a9c81b07b703b609733, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1c504ae1e6184d7aa51f7c7c1152945f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/829eb1a2551544058e98bd4fb3f17cf6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1d0ea2acf02c4ea49220f30d64df91f6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/30e602c490624a25a642024ba0345e4f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/7c0ed1ecfc474cfa93f247ab496765f5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/9839eabd0f794958805f38ad4bed6331, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8acfd920e9f468580bd8cf8ec74c0a2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/60bf0476484a401a971a6100b5cefa9d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7bef3f698049dfb2add9a0abb734d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ac8cc5a97c6e487fb9e589bb4816d4f5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4bdb94c5dc6e4121b92c5f39ce3090c3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/51901d7d91f6401bb788942200414ab6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8b7fb6f809740fbad74d1fec99e750a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ef038ec71aa14036b356ba430cc7373c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2baf06d5a9a549e0acb05b260a56db13, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/71d2db42c1de41cc828a8205132b11f8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/164d76250bc44e75b8c9f17e8d62208b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5de63ae2139349509734bd9d00a2d30b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d03335862ef04362a035f2eb4042823e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3f3247791bd74998b7799bd31a0e4f57, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/8d8b27b0959146aa8d31b81378b13b6b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/6d895d6f703a415280e9e9ae97db013d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/95782a33db2148d6a52360c649385f16, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/06921fb122214489b31bd19c43ccdeb2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/b08f1cb9960542189af344da67e412d5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3c45bb91f26a49b0898f987d0ae71ef6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5f1b3aad8d8643d9911e533cdf5915c9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4177e5701beb4bd5bafea4e4c39ef789, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/fd5f78d2d91d467a84b55743c1c2f78f] to archive 2024-11-13T05:50:47,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:50:47,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/de241b95421546b88ee089ed81cf39cb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/de241b95421546b88ee089ed81cf39cb 2024-11-13T05:50:47,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1a01420dc631470b9926d92ee6c3a26d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1a01420dc631470b9926d92ee6c3a26d 2024-11-13T05:50:47,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d8050035cdbb47eab2026905ab489594 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d8050035cdbb47eab2026905ab489594 2024-11-13T05:50:47,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/277ed23bd572469690923c228eebe18d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/277ed23bd572469690923c228eebe18d 2024-11-13T05:50:47,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/c8828ed585f44a9c81b07b703b609733 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/c8828ed585f44a9c81b07b703b609733 2024-11-13T05:50:47,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1c504ae1e6184d7aa51f7c7c1152945f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1c504ae1e6184d7aa51f7c7c1152945f 2024-11-13T05:50:47,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/829eb1a2551544058e98bd4fb3f17cf6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/829eb1a2551544058e98bd4fb3f17cf6 2024-11-13T05:50:47,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1d0ea2acf02c4ea49220f30d64df91f6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/1d0ea2acf02c4ea49220f30d64df91f6 2024-11-13T05:50:47,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/30e602c490624a25a642024ba0345e4f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/30e602c490624a25a642024ba0345e4f 2024-11-13T05:50:47,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/7c0ed1ecfc474cfa93f247ab496765f5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/7c0ed1ecfc474cfa93f247ab496765f5 2024-11-13T05:50:47,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/9839eabd0f794958805f38ad4bed6331 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/9839eabd0f794958805f38ad4bed6331 2024-11-13T05:50:47,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8acfd920e9f468580bd8cf8ec74c0a2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8acfd920e9f468580bd8cf8ec74c0a2 2024-11-13T05:50:47,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/60bf0476484a401a971a6100b5cefa9d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/60bf0476484a401a971a6100b5cefa9d 2024-11-13T05:50:47,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7bef3f698049dfb2add9a0abb734d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7bef3f698049dfb2add9a0abb734d9 2024-11-13T05:50:47,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ac8cc5a97c6e487fb9e589bb4816d4f5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ac8cc5a97c6e487fb9e589bb4816d4f5 2024-11-13T05:50:47,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4bdb94c5dc6e4121b92c5f39ce3090c3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4bdb94c5dc6e4121b92c5f39ce3090c3 2024-11-13T05:50:47,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/51901d7d91f6401bb788942200414ab6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/51901d7d91f6401bb788942200414ab6 2024-11-13T05:50:47,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/dff9ed05534f4d7cbf5a3d5e6b4a3b61 2024-11-13T05:50:47,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8b7fb6f809740fbad74d1fec99e750a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/e8b7fb6f809740fbad74d1fec99e750a 2024-11-13T05:50:47,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ef038ec71aa14036b356ba430cc7373c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ef038ec71aa14036b356ba430cc7373c 2024-11-13T05:50:47,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2baf06d5a9a549e0acb05b260a56db13 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2baf06d5a9a549e0acb05b260a56db13 2024-11-13T05:50:47,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/71d2db42c1de41cc828a8205132b11f8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/71d2db42c1de41cc828a8205132b11f8 2024-11-13T05:50:47,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/164d76250bc44e75b8c9f17e8d62208b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/164d76250bc44e75b8c9f17e8d62208b 2024-11-13T05:50:47,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5de63ae2139349509734bd9d00a2d30b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5de63ae2139349509734bd9d00a2d30b 2024-11-13T05:50:47,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d03335862ef04362a035f2eb4042823e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/d03335862ef04362a035f2eb4042823e 2024-11-13T05:50:47,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3f3247791bd74998b7799bd31a0e4f57 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3f3247791bd74998b7799bd31a0e4f57 2024-11-13T05:50:47,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/8d8b27b0959146aa8d31b81378b13b6b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/8d8b27b0959146aa8d31b81378b13b6b 2024-11-13T05:50:47,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/6d895d6f703a415280e9e9ae97db013d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/6d895d6f703a415280e9e9ae97db013d 2024-11-13T05:50:47,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/95782a33db2148d6a52360c649385f16 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/95782a33db2148d6a52360c649385f16 2024-11-13T05:50:47,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/06921fb122214489b31bd19c43ccdeb2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/06921fb122214489b31bd19c43ccdeb2 2024-11-13T05:50:47,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/b08f1cb9960542189af344da67e412d5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/b08f1cb9960542189af344da67e412d5 2024-11-13T05:50:47,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3c45bb91f26a49b0898f987d0ae71ef6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/3c45bb91f26a49b0898f987d0ae71ef6 2024-11-13T05:50:47,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5f1b3aad8d8643d9911e533cdf5915c9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/5f1b3aad8d8643d9911e533cdf5915c9 2024-11-13T05:50:47,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4177e5701beb4bd5bafea4e4c39ef789 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/4177e5701beb4bd5bafea4e4c39ef789 2024-11-13T05:50:47,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/fd5f78d2d91d467a84b55743c1c2f78f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/fd5f78d2d91d467a84b55743c1c2f78f 2024-11-13T05:50:47,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fbbcf7a3f3674961b3e7cf6e765f08c8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6da2853fa141450783a190d8210fdffe, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5c365f00df464bd0b9a00c9b8f120bfe, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5ca7980182de4c459eb7abbd55c0cff1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1891309586394b7797529476cc441def, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/7b1f0fd615224c6b8a91e7a45efeaa17, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/086166cb28984d5789328122b700b223, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/85179127be49468a940d4279d85753cb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d6ecd69dbfe14439b0666324f69d9576, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/ede10a9d753144b593f015eeb710c5d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5db667aba8b94098a087bc9aa4664fa5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/3b883becb99f48a18c9bb9d213efedfa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/477d6468ec4f465ba682d67e971296ff, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/eb764321e567415ba84c103d6c0a2193, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5d38b2eef0214e8fb1eb081eff4e22c6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fcc3fed020434ec497c884f0042e3c30, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/a83e3b874d6148b49b4c16618896a054, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/814f5ee504514f40a44895c3da248781, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1739b97f2fb848a79516ef482fb4b73d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/250d3bc15b564b9c8883a13c2bba5fca, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/b7a167c71c134246924a032cfd66e3f9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/e97c1d27f7bf47b3900c3c313e6989c7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/4eaf8f3c962348cf88ab1f87eca25c81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/40ee76ee023744b58caa6f8a48e24e68, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fdd3b2e554424fb9b2ae6cb545ea0c1c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/27eb5010c28e4ff0b2a488e4dc1aaae3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/06f04e5879c841c5a9ab749ef266872a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fa856e55a4a541d989d6ecb6bf3f1fc5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/07f49fe691b74a6da438037d6ed6d7af, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d4b92b96551e48178fdaa0a83783e60c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1c61729146c94b0da01e9096dbec56b8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/cd64a140245d4026a44053264be5503a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6ad42e76dedd4e41a0db3ca4f81bb0ed, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6222a0962cc44dacb1ffe92acc3ee0d5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d658aedf73784f4c95b1bdee63036ebe] to archive 2024-11-13T05:50:47,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:50:47,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fbbcf7a3f3674961b3e7cf6e765f08c8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fbbcf7a3f3674961b3e7cf6e765f08c8 2024-11-13T05:50:47,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6da2853fa141450783a190d8210fdffe to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6da2853fa141450783a190d8210fdffe 2024-11-13T05:50:47,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5c365f00df464bd0b9a00c9b8f120bfe to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5c365f00df464bd0b9a00c9b8f120bfe 2024-11-13T05:50:47,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5ca7980182de4c459eb7abbd55c0cff1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5ca7980182de4c459eb7abbd55c0cff1 2024-11-13T05:50:47,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1891309586394b7797529476cc441def to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1891309586394b7797529476cc441def 2024-11-13T05:50:47,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/7b1f0fd615224c6b8a91e7a45efeaa17 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/7b1f0fd615224c6b8a91e7a45efeaa17 2024-11-13T05:50:47,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/086166cb28984d5789328122b700b223 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/086166cb28984d5789328122b700b223 2024-11-13T05:50:47,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/85179127be49468a940d4279d85753cb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/85179127be49468a940d4279d85753cb 2024-11-13T05:50:47,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d6ecd69dbfe14439b0666324f69d9576 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d6ecd69dbfe14439b0666324f69d9576 2024-11-13T05:50:47,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/ede10a9d753144b593f015eeb710c5d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/ede10a9d753144b593f015eeb710c5d9 2024-11-13T05:50:47,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5db667aba8b94098a087bc9aa4664fa5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5db667aba8b94098a087bc9aa4664fa5 2024-11-13T05:50:47,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/3b883becb99f48a18c9bb9d213efedfa to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/3b883becb99f48a18c9bb9d213efedfa 2024-11-13T05:50:47,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/477d6468ec4f465ba682d67e971296ff to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/477d6468ec4f465ba682d67e971296ff 2024-11-13T05:50:47,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/eb764321e567415ba84c103d6c0a2193 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/eb764321e567415ba84c103d6c0a2193 2024-11-13T05:50:47,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5d38b2eef0214e8fb1eb081eff4e22c6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/5d38b2eef0214e8fb1eb081eff4e22c6 2024-11-13T05:50:47,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fcc3fed020434ec497c884f0042e3c30 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fcc3fed020434ec497c884f0042e3c30 2024-11-13T05:50:47,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/a83e3b874d6148b49b4c16618896a054 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/a83e3b874d6148b49b4c16618896a054 2024-11-13T05:50:47,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/814f5ee504514f40a44895c3da248781 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/814f5ee504514f40a44895c3da248781 2024-11-13T05:50:47,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1739b97f2fb848a79516ef482fb4b73d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1739b97f2fb848a79516ef482fb4b73d 2024-11-13T05:50:47,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/250d3bc15b564b9c8883a13c2bba5fca to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/250d3bc15b564b9c8883a13c2bba5fca 2024-11-13T05:50:47,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/b7a167c71c134246924a032cfd66e3f9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/b7a167c71c134246924a032cfd66e3f9 2024-11-13T05:50:47,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/e97c1d27f7bf47b3900c3c313e6989c7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/e97c1d27f7bf47b3900c3c313e6989c7 2024-11-13T05:50:47,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/4eaf8f3c962348cf88ab1f87eca25c81 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/4eaf8f3c962348cf88ab1f87eca25c81 2024-11-13T05:50:47,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/40ee76ee023744b58caa6f8a48e24e68 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/40ee76ee023744b58caa6f8a48e24e68 2024-11-13T05:50:47,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fdd3b2e554424fb9b2ae6cb545ea0c1c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fdd3b2e554424fb9b2ae6cb545ea0c1c 2024-11-13T05:50:47,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/27eb5010c28e4ff0b2a488e4dc1aaae3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/27eb5010c28e4ff0b2a488e4dc1aaae3 2024-11-13T05:50:47,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/06f04e5879c841c5a9ab749ef266872a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/06f04e5879c841c5a9ab749ef266872a 2024-11-13T05:50:47,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fa856e55a4a541d989d6ecb6bf3f1fc5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/fa856e55a4a541d989d6ecb6bf3f1fc5 2024-11-13T05:50:47,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/07f49fe691b74a6da438037d6ed6d7af to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/07f49fe691b74a6da438037d6ed6d7af 2024-11-13T05:50:47,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d4b92b96551e48178fdaa0a83783e60c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d4b92b96551e48178fdaa0a83783e60c 2024-11-13T05:50:47,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1c61729146c94b0da01e9096dbec56b8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/1c61729146c94b0da01e9096dbec56b8 2024-11-13T05:50:47,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/cd64a140245d4026a44053264be5503a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/cd64a140245d4026a44053264be5503a 2024-11-13T05:50:47,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6ad42e76dedd4e41a0db3ca4f81bb0ed to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6ad42e76dedd4e41a0db3ca4f81bb0ed 2024-11-13T05:50:47,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6222a0962cc44dacb1ffe92acc3ee0d5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/6222a0962cc44dacb1ffe92acc3ee0d5 2024-11-13T05:50:47,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d658aedf73784f4c95b1bdee63036ebe to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/d658aedf73784f4c95b1bdee63036ebe 2024-11-13T05:50:47,404 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/recovered.edits/518.seqid, newMaxSeqId=518, maxSeqId=1 2024-11-13T05:50:47,405 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. 2024-11-13T05:50:47,405 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 9480a359b149cb565364fd0f1f82db91: Waiting for close lock at 1731477045963Running coprocessor pre-close hooks at 1731477045963Disabling compacts and flushes for region at 1731477045963Disabling writes for close at 1731477045964 (+1 ms)Obtaining lock to block concurrent updates at 1731477045964Preparing flush snapshotting stores in 9480a359b149cb565364fd0f1f82db91 at 1731477045964Finished memstore snapshotting TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731477045965 (+1 ms)Flushing stores of TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91. at 1731477045966 (+1 ms)Flushing 9480a359b149cb565364fd0f1f82db91/A: creating writer at 1731477045966Flushing 9480a359b149cb565364fd0f1f82db91/A: appending metadata at 1731477045973 (+7 ms)Flushing 9480a359b149cb565364fd0f1f82db91/A: closing flushed file at 1731477045974 (+1 ms)Flushing 9480a359b149cb565364fd0f1f82db91/B: creating writer at 1731477046386 (+412 ms)Flushing 9480a359b149cb565364fd0f1f82db91/B: appending metadata at 1731477046390 (+4 ms)Flushing 9480a359b149cb565364fd0f1f82db91/B: closing flushed file at 1731477046391 (+1 ms)Flushing 9480a359b149cb565364fd0f1f82db91/C: creating writer at 1731477046806 (+415 ms)Flushing 9480a359b149cb565364fd0f1f82db91/C: appending metadata at 1731477046810 (+4 ms)Flushing 9480a359b149cb565364fd0f1f82db91/C: closing flushed file at 1731477046811 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a62a78: reopening flushed file at 1731477047221 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289b264a: reopening flushed file at 1731477047227 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11d9a781: reopening flushed file at 1731477047232 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9480a359b149cb565364fd0f1f82db91 in 1273ms, sequenceid=515, compaction requested=true at 1731477047237 (+5 ms)Writing region close event to WAL at 1731477047400 (+163 ms)Running coprocessor post-close hooks at 1731477047405 (+5 ms)Closed at 1731477047405 2024-11-13T05:50:47,408 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:47,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9480a359b149cb565364fd0f1f82db91, regionState=CLOSED 2024-11-13T05:50:47,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:50:47,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T05:50:47,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 9480a359b149cb565364fd0f1f82db91, server=bc386b7bf2fe,38979,1731477016232 in 1.6100 sec 2024-11-13T05:50:47,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T05:50:47,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9480a359b149cb565364fd0f1f82db91, UNASSIGN in 1.6180 sec 2024-11-13T05:50:47,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T05:50:47,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6260 sec 2024-11-13T05:50:47,422 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477047422"}]},"ts":"1731477047422"} 2024-11-13T05:50:47,424 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:50:47,424 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:50:47,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6430 sec 2024-11-13T05:50:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T05:50:47,942 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:47,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:50:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:47,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T05:50:47,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:47,958 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:47,962 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/recovered.edits] 2024-11-13T05:50:47,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/2d2f5b6bae23456d9905be11074c430f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/2d2f5b6bae23456d9905be11074c430f 2024-11-13T05:50:47,967 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9e435d19f6ff4b0f9d8a83c08e16dafd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/9e435d19f6ff4b0f9d8a83c08e16dafd 2024-11-13T05:50:47,969 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fbe0a81e7f0844c594671a0f0712b6f3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/A/fbe0a81e7f0844c594671a0f0712b6f3 2024-11-13T05:50:47,971 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2c7864cc3b0345dabb5bb7c85c159cfb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/2c7864cc3b0345dabb5bb7c85c159cfb 2024-11-13T05:50:47,972 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/84530f0680444f67b0933be12705ae52 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/84530f0680444f67b0933be12705ae52 2024-11-13T05:50:47,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7da4d975824e7480f95dec510b2f4c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/B/ad7da4d975824e7480f95dec510b2f4c 2024-11-13T05:50:47,975 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/064b303bb81c4517953d489d735d8455 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/064b303bb81c4517953d489d735d8455 2024-11-13T05:50:47,976 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/39f7dfaa18ce499da14558576541b94d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/39f7dfaa18ce499da14558576541b94d 2024-11-13T05:50:47,977 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/84410c48565c48e9930e2dde6a19914a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/C/84410c48565c48e9930e2dde6a19914a 2024-11-13T05:50:47,979 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/recovered.edits/518.seqid to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91/recovered.edits/518.seqid 2024-11-13T05:50:47,980 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/9480a359b149cb565364fd0f1f82db91 2024-11-13T05:50:47,980 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:50:47,984 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:47,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T05:50:47,993 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:50:48,020 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:50:48,021 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:48,021 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:50:48,021 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731477048021"}]},"ts":"9223372036854775807"} 2024-11-13T05:50:48,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:50:48,025 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9480a359b149cb565364fd0f1f82db91, NAME => 'TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:50:48,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:50:48,026 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731477048026"}]},"ts":"9223372036854775807"} 2024-11-13T05:50:48,028 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:50:48,029 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:48,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-13T05:50:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T05:50:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:50:48,062 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:48,077 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RS:0;bc386b7bf2fe:38979-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1749926084_22 at /127.0.0.1:33216 [Waiting for operation #259] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1749926084_22 at /127.0.0.1:58336 [Waiting for operation #275] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-820210548_22 at /127.0.0.1:35360 [Waiting for operation #184] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=459 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7133 (was 7132) - AvailableMemoryMB LEAK? - 2024-11-13T05:50:48,086 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=459, ProcessCount=11, AvailableMemoryMB=7132 2024-11-13T05:50:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:50:48,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:50:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:48,090 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:50:48,090 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:48,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-13T05:50:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:50:48,091 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:50:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741951_1127 (size=1061) 2024-11-13T05:50:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:50:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:50:48,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:50:48,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741952_1128 (size=53) 2024-11-13T05:50:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:50:48,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:48,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5eaaa9d25e2cd5c3d12b429e77fa393b, disabling compactions & flushes 2024-11-13T05:50:48,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:48,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:48,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. after waiting 0 ms 2024-11-13T05:50:48,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:48,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:48,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: Waiting for close lock at 1731477048906Disabling compacts and flushes for region at 1731477048906Disabling writes for close at 1731477048906Writing region close event to WAL at 1731477048906Closed at 1731477048906 2024-11-13T05:50:48,908 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:50:48,908 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731477048908"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477048908"}]},"ts":"1731477048908"} 2024-11-13T05:50:48,910 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:50:48,912 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:50:48,912 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477048912"}]},"ts":"1731477048912"} 2024-11-13T05:50:48,914 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:50:48,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, ASSIGN}] 2024-11-13T05:50:48,916 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, ASSIGN 2024-11-13T05:50:48,916 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=false 2024-11-13T05:50:49,067 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:49,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, ASSIGN because future has completed 2024-11-13T05:50:49,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:50:49,227 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:49,227 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:50:49,227 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,227 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:49,228 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,228 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,229 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,231 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:50:49,232 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaaa9d25e2cd5c3d12b429e77fa393b columnFamilyName A 2024-11-13T05:50:49,233 DEBUG [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:49,234 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(327): Store=5eaaa9d25e2cd5c3d12b429e77fa393b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:49,234 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,236 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:49,236 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaaa9d25e2cd5c3d12b429e77fa393b columnFamilyName B 2024-11-13T05:50:49,236 DEBUG [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:49,237 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(327): Store=5eaaa9d25e2cd5c3d12b429e77fa393b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:49,237 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,239 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:49,239 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaaa9d25e2cd5c3d12b429e77fa393b columnFamilyName C 2024-11-13T05:50:49,239 DEBUG [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:49,239 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(327): Store=5eaaa9d25e2cd5c3d12b429e77fa393b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:49,240 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:49,240 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,240 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,241 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,242 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,242 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,242 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:50:49,243 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,245 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:50:49,245 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5eaaa9d25e2cd5c3d12b429e77fa393b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72263358, jitterRate=0.07680794596672058}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:50:49,245 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:49,246 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: Running coprocessor pre-open hook at 1731477049228Writing region info on filesystem at 1731477049228Initializing all the Stores at 1731477049229 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477049229Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477049229Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477049229Cleaning up temporary data from old regions at 1731477049242 (+13 ms)Running coprocessor post-open hooks at 1731477049245 (+3 ms)Region opened successfully at 1731477049246 (+1 ms) 2024-11-13T05:50:49,247 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., pid=34, masterSystemTime=1731477049223 2024-11-13T05:50:49,248 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:49,248 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:49,249 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:49,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:50:49,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-13T05:50:49,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 in 182 msec 2024-11-13T05:50:49,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T05:50:49,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, ASSIGN in 340 msec 2024-11-13T05:50:49,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:50:49,257 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477049257"}]},"ts":"1731477049257"} 2024-11-13T05:50:49,259 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:50:49,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:50:49,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-13T05:50:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-13T05:50:50,251 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:50,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ff43b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:50,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:50,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:50,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63097a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:50,256 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655d0d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:50,256 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:50,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:50,260 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:50:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:50:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ed75f2b 2024-11-13T05:50:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:50:50,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:50:50,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:50:50,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:50:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T05:50:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741953_1129 (size=1097) 2024-11-13T05:50:50,689 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T05:50:50,689 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T05:50:50,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:50:50,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, REOPEN/MOVE}] 2024-11-13T05:50:50,701 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, REOPEN/MOVE 2024-11-13T05:50:50,701 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:50,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, REOPEN/MOVE because future has completed 2024-11-13T05:50:50,703 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:50:50,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:50:50,856 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:50,856 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:50:50,856 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 5eaaa9d25e2cd5c3d12b429e77fa393b, disabling compactions & flushes 2024-11-13T05:50:50,856 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:50,856 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:50,856 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. after waiting 0 ms 2024-11-13T05:50:50,856 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:50,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:50:50,861 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:50,861 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: Waiting for close lock at 1731477050856Running coprocessor pre-close hooks at 1731477050856Disabling compacts and flushes for region at 1731477050856Disabling writes for close at 1731477050856Writing region close event to WAL at 1731477050857 (+1 ms)Running coprocessor post-close hooks at 1731477050860 (+3 ms)Closed at 1731477050861 (+1 ms) 2024-11-13T05:50:50,861 WARN [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 5eaaa9d25e2cd5c3d12b429e77fa393b to self. 2024-11-13T05:50:50,863 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:50,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=CLOSED 2024-11-13T05:50:50,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:50:50,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-13T05:50:50,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 in 164 msec 2024-11-13T05:50:50,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=true 2024-11-13T05:50:51,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, REOPEN/MOVE because future has completed 2024-11-13T05:50:51,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:50:51,177 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,177 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:50:51,178 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,178 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:50:51,178 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,178 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,180 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,181 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:51,184 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaaa9d25e2cd5c3d12b429e77fa393b columnFamilyName A 2024-11-13T05:50:51,186 DEBUG [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:51,187 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(327): Store=5eaaa9d25e2cd5c3d12b429e77fa393b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:51,187 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,188 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:51,188 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaaa9d25e2cd5c3d12b429e77fa393b columnFamilyName B 2024-11-13T05:50:51,188 DEBUG [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:51,188 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(327): Store=5eaaa9d25e2cd5c3d12b429e77fa393b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:51,188 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,189 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:50:51,189 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaaa9d25e2cd5c3d12b429e77fa393b columnFamilyName C 2024-11-13T05:50:51,189 DEBUG [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:51,189 INFO [StoreOpener-5eaaa9d25e2cd5c3d12b429e77fa393b-1 {}] regionserver.HStore(327): Store=5eaaa9d25e2cd5c3d12b429e77fa393b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:50:51,190 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,190 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,190 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,191 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,192 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,192 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,193 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:50:51,194 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,195 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5eaaa9d25e2cd5c3d12b429e77fa393b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68989239, jitterRate=0.028019770979881287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:50:51,195 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,196 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: Running coprocessor pre-open hook at 1731477051178Writing region info on filesystem at 1731477051178Initializing all the Stores at 1731477051179 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477051179Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477051179Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477051179Cleaning up temporary data from old regions at 1731477051192 (+13 ms)Running coprocessor post-open hooks at 1731477051195 (+3 ms)Region opened successfully at 1731477051196 (+1 ms) 2024-11-13T05:50:51,196 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., pid=39, masterSystemTime=1731477051174 2024-11-13T05:50:51,198 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,198 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,199 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:50:51,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-13T05:50:51,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 in 179 msec 2024-11-13T05:50:51,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T05:50:51,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, REOPEN/MOVE in 504 msec 2024-11-13T05:50:51,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-13T05:50:51,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-13T05:50:51,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-13T05:50:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-13T05:50:51,213 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:51,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,214 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50eea465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,216 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0451e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0133ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,218 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17322130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,219 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433e5e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9032c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587ea6a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d415392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,227 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed3501e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,228 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4c9bdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c562e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,230 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b6c48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,232 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18b66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adec721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,234 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2568fef5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,236 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7905a199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c2fc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,239 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64877518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,241 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ac8dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6724e437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,243 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6f899f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5fc009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1801a130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:50:51,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:50:51,247 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:50:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:50:51,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:50:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4baf1f77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:50:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:50:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:50:51,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:50:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:50:51,250 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,250 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,251 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,252 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,253 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,253 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,254 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,256 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:51,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T05:50:51,257 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:50:51,259 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,259 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:50:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,261 DEBUG [Thread-658 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:51,261 DEBUG [Thread-660 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:50:51,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:51,261 DEBUG [Thread-658 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,262 DEBUG [Thread-660 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:50:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:50:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:50:51,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:50:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731477051320, max timestamp=1731477051340Num uniques 859; 2024-11-13T05:50:51,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T05:50:51,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111336b4a16cfdd14d06b55a4d0e448c4cee_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477051271/Put/seqid=0 2024-11-13T05:50:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:50:51,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731477051320, max timestamp=1731477051340Num uniques 859; 2024-11-13T05:50:51,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T05:50:51,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731477051320, max timestamp=1731477051340Num uniques 859; 2024-11-13T05:50:51,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-13T05:50:51,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741954_1130 (size=14594) 2024-11-13T05:50:51,373 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:51,380 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111336b4a16cfdd14d06b55a4d0e448c4cee_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111336b4a16cfdd14d06b55a4d0e448c4cee_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:51,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/af09412b35354aa48e2c94059056fb81, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:51,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/af09412b35354aa48e2c94059056fb81 is 175, key is test_row_0/A:col10/1731477051271/Put/seqid=0 2024-11-13T05:50:51,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741955_1131 (size=39549) 2024-11-13T05:50:51,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/af09412b35354aa48e2c94059056fb81 2024-11-13T05:50:51,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477111417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477111427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477111427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477111430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477111433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/563a52581ccf415d97d0f03c872eb39a is 50, key is test_row_0/B:col10/1731477051271/Put/seqid=0 2024-11-13T05:50:51,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741956_1132 (size=12001) 2024-11-13T05:50:51,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/563a52581ccf415d97d0f03c872eb39a 2024-11-13T05:50:51,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477111543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477111563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477111563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:50:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477111563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477111572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/71f595a214254acc982d5f3cbd879020 is 50, key is test_row_0/C:col10/1731477051271/Put/seqid=0 2024-11-13T05:50:51,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741957_1133 (size=12001) 2024-11-13T05:50:51,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:51,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:51,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477111762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477111782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477111782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477111772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477111802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:50:51,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:51,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/71f595a214254acc982d5f3cbd879020 2024-11-13T05:50:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/af09412b35354aa48e2c94059056fb81 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81 2024-11-13T05:50:52,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:52,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81, entries=200, sequenceid=14, filesize=38.6 K 2024-11-13T05:50:52,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/563a52581ccf415d97d0f03c872eb39a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/563a52581ccf415d97d0f03c872eb39a 2024-11-13T05:50:52,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/563a52581ccf415d97d0f03c872eb39a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:50:52,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/71f595a214254acc982d5f3cbd879020 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/71f595a214254acc982d5f3cbd879020 2024-11-13T05:50:52,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/71f595a214254acc982d5f3cbd879020, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:50:52,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 798ms, sequenceid=14, compaction requested=false 2024-11-13T05:50:52,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:52,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-13T05:50:52,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:52,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:52,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:52,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:52,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:52,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477112117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477112119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133fc079fa49fd444c81bce7075ef4a513_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477051428/Put/seqid=0 2024-11-13T05:50:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477112122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477112123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477112127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741958_1134 (size=14594) 2024-11-13T05:50:52,154 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:52,160 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133fc079fa49fd444c81bce7075ef4a513_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133fc079fa49fd444c81bce7075ef4a513_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:52,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/7aa6200fa8334302902158378ba550c4, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:52,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/7aa6200fa8334302902158378ba550c4 is 175, key is test_row_0/A:col10/1731477051428/Put/seqid=0 2024-11-13T05:50:52,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741959_1135 (size=39549) 2024-11-13T05:50:52,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/7aa6200fa8334302902158378ba550c4 2024-11-13T05:50:52,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:52,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:52,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/47588702f9c948078717bad851c29bcf is 50, key is test_row_0/B:col10/1731477051428/Put/seqid=0 2024-11-13T05:50:52,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477112231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477112232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477112243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477112241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741960_1136 (size=12001) 2024-11-13T05:50:52,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/47588702f9c948078717bad851c29bcf 2024-11-13T05:50:52,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/a14eb3d7205e486884b6df46136585c5 is 50, key is test_row_0/C:col10/1731477051428/Put/seqid=0 2024-11-13T05:50:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741961_1137 (size=12001) 2024-11-13T05:50:52,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/a14eb3d7205e486884b6df46136585c5 2024-11-13T05:50:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/7aa6200fa8334302902158378ba550c4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4 2024-11-13T05:50:52,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4, entries=200, sequenceid=49, filesize=38.6 K 2024-11-13T05:50:52,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/47588702f9c948078717bad851c29bcf as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/47588702f9c948078717bad851c29bcf 2024-11-13T05:50:52,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/47588702f9c948078717bad851c29bcf, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T05:50:52,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/a14eb3d7205e486884b6df46136585c5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/a14eb3d7205e486884b6df46136585c5 2024-11-13T05:50:52,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/a14eb3d7205e486884b6df46136585c5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T05:50:52,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 236ms, sequenceid=49, compaction requested=false 2024-11-13T05:50:52,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:52,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T05:50:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:52,361 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T05:50:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113563b4e9f591e4c5198012e68fe8ed439_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477052113/Put/seqid=0 2024-11-13T05:50:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:50:52,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741962_1138 (size=12154) 2024-11-13T05:50:52,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:52,428 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113563b4e9f591e4c5198012e68fe8ed439_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113563b4e9f591e4c5198012e68fe8ed439_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/45808d2342e24953b38e460d869d73d2, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/45808d2342e24953b38e460d869d73d2 is 175, key is test_row_0/A:col10/1731477052113/Put/seqid=0 2024-11-13T05:50:52,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:52,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741963_1139 (size=30955) 2024-11-13T05:50:52,469 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/45808d2342e24953b38e460d869d73d2 2024-11-13T05:50:52,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/647e901813ac40babed06816291a2c6e is 50, key is test_row_0/B:col10/1731477052113/Put/seqid=0 2024-11-13T05:50:52,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741964_1140 (size=12001) 2024-11-13T05:50:52,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477052529, max timestamp=1731477052544Num uniques 859; 2024-11-13T05:50:52,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:50:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477052529, max timestamp=1731477052544Num uniques 859; 2024-11-13T05:50:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477052529, max timestamp=1731477052544Num uniques 859; 2024-11-13T05:50:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:50:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:50:52,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477112618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477112621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477112623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477112626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477112642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477112731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477112731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477112742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477112742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,905 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/647e901813ac40babed06816291a2c6e 2024-11-13T05:50:52,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/b690e14b23b7431584e7e74578a7595d is 50, key is test_row_0/C:col10/1731477052113/Put/seqid=0 2024-11-13T05:50:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741965_1141 (size=12001) 2024-11-13T05:50:52,933 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/b690e14b23b7431584e7e74578a7595d 2024-11-13T05:50:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/45808d2342e24953b38e460d869d73d2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2 2024-11-13T05:50:52,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477112942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477112945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,948 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2, entries=150, sequenceid=58, filesize=30.2 K 2024-11-13T05:50:52,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/647e901813ac40babed06816291a2c6e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/647e901813ac40babed06816291a2c6e 2024-11-13T05:50:52,959 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:50:52,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477112957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,971 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/647e901813ac40babed06816291a2c6e, entries=150, sequenceid=58, filesize=11.7 K 2024-11-13T05:50:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477112962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:52,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/b690e14b23b7431584e7e74578a7595d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b690e14b23b7431584e7e74578a7595d 2024-11-13T05:50:52,992 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b690e14b23b7431584e7e74578a7595d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-13T05:50:52,994 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 633ms, sequenceid=58, compaction requested=true 2024-11-13T05:50:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T05:50:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T05:50:53,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T05:50:53,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7360 sec 2024-11-13T05:50:53,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7450 sec 2024-11-13T05:50:53,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477052607, max timestamp=1731477053257Num uniques 1068; 2024-11-13T05:50:53,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T05:50:53,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477052607, max timestamp=1731477053257Num uniques 1068; 2024-11-13T05:50:53,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-13T05:50:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:53,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-11-13T05:50:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:53,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:50:53,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113531b404a2498491b88698c5083eeb946_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741966_1142 (size=14594) 2024-11-13T05:50:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477113317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477113319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477113321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477113321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T05:50:53,402 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:53,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T05:50:53,405 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:53,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:53,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:53,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477113432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477113432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477113433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477113433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:53,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477113642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477113642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477113642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477113642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477113663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:53,668 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:53,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:53,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113531b404a2498491b88698c5083eeb946_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113531b404a2498491b88698c5083eeb946_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:53,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/08f777ca282641d1b4e6ffb14d809753, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:53,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/08f777ca282641d1b4e6ffb14d809753 is 175, key is test_row_0/A:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:53,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741967_1143 (size=39549) 2024-11-13T05:50:53,763 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/08f777ca282641d1b4e6ffb14d809753 2024-11-13T05:50:53,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/578b2ac2c4a94a64b65fc5fe61a2816f is 50, key is test_row_0/B:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:53,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741968_1144 (size=12001) 2024-11-13T05:50:53,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/578b2ac2c4a94a64b65fc5fe61a2816f 2024-11-13T05:50:53,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/fb9b71f7c5a1416eb5c5415e18367439 is 50, key is test_row_0/C:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:53,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:53,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:53,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741969_1145 (size=12001) 2024-11-13T05:50:53,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/fb9b71f7c5a1416eb5c5415e18367439 2024-11-13T05:50:53,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/08f777ca282641d1b4e6ffb14d809753 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753 2024-11-13T05:50:53,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753, entries=200, sequenceid=96, filesize=38.6 K 2024-11-13T05:50:53,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/578b2ac2c4a94a64b65fc5fe61a2816f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/578b2ac2c4a94a64b65fc5fe61a2816f 2024-11-13T05:50:53,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/578b2ac2c4a94a64b65fc5fe61a2816f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T05:50:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/fb9b71f7c5a1416eb5c5415e18367439 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/fb9b71f7c5a1416eb5c5415e18367439 2024-11-13T05:50:53,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/fb9b71f7c5a1416eb5c5415e18367439, entries=150, sequenceid=96, filesize=11.7 K 2024-11-13T05:50:53,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~183.83 KB/188240, currentSize=114.05 KB/116790 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 690ms, sequenceid=96, compaction requested=true 2024-11-13T05:50:53,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:53,953 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:53,953 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:53,955 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:53,956 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:50:53,956 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,955 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:53,956 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:50:53,956 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,956 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/563a52581ccf415d97d0f03c872eb39a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/47588702f9c948078717bad851c29bcf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/647e901813ac40babed06816291a2c6e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/578b2ac2c4a94a64b65fc5fe61a2816f] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=46.9 K 2024-11-13T05:50:53,956 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=146.1 K 2024-11-13T05:50:53,957 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:53,957 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753] 2024-11-13T05:50:53,960 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 563a52581ccf415d97d0f03c872eb39a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477051271 2024-11-13T05:50:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:53,960 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 47588702f9c948078717bad851c29bcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477051422 2024-11-13T05:50:53,961 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting af09412b35354aa48e2c94059056fb81, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477051268 2024-11-13T05:50:53,961 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 647e901813ac40babed06816291a2c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731477052113 2024-11-13T05:50:53,961 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa6200fa8334302902158378ba550c4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477051422 2024-11-13T05:50:53,961 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 578b2ac2c4a94a64b65fc5fe61a2816f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731477052620 2024-11-13T05:50:53,961 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45808d2342e24953b38e460d869d73d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731477052113 2024-11-13T05:50:53,962 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f777ca282641d1b4e6ffb14d809753, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731477052607 2024-11-13T05:50:53,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477053300, max timestamp=1731477053963Num uniques 859; 2024-11-13T05:50:53,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:53,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477053300, max timestamp=1731477053963Num uniques 859; 2024-11-13T05:50:53,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:53,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477053300, max timestamp=1731477053963Num uniques 859; 2024-11-13T05:50:53,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:53,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-13T05:50:53,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:53,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:53,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:53,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:53,991 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:54,004 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#127 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:54,004 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5d5f5986213d4d779dbf3434575d0c08 is 50, key is test_row_0/B:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:54,026 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fd270f056021458689ca69e226eccca2_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:54,035 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fd270f056021458689ca69e226eccca2_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:54,035 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fd270f056021458689ca69e226eccca2_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:54,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391d05a0610d74e27a245b63210f69cab_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477053963/Put/seqid=0 2024-11-13T05:50:54,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477053999, max timestamp=1731477054052Num uniques 859; 2024-11-13T05:50:54,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:50:54,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477053999, max timestamp=1731477054052Num uniques 859; 2024-11-13T05:50:54,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:50:54,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477053999, max timestamp=1731477054052Num uniques 859; 2024-11-13T05:50:54,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:50:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741970_1146 (size=12139) 2024-11-13T05:50:54,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741972_1148 (size=17034) 2024-11-13T05:50:54,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741971_1147 (size=4469) 2024-11-13T05:50:54,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477114152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477114155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477114157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477114157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477114272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477114272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477114272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477114273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,484 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5d5f5986213d4d779dbf3434575d0c08 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5d5f5986213d4d779dbf3434575d0c08 2024-11-13T05:50:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477114481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477114482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477114482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477114482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,498 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into 5d5f5986213d4d779dbf3434575d0c08(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:54,498 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:54,498 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=12, startTime=1731477053953; duration=0sec 2024-11-13T05:50:54,498 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:54,498 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:50:54,498 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:54,500 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:54,501 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:50:54,501 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,501 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/71f595a214254acc982d5f3cbd879020, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/a14eb3d7205e486884b6df46136585c5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b690e14b23b7431584e7e74578a7595d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/fb9b71f7c5a1416eb5c5415e18367439] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=46.9 K 2024-11-13T05:50:54,501 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 71f595a214254acc982d5f3cbd879020, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477051271 2024-11-13T05:50:54,502 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a14eb3d7205e486884b6df46136585c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477051422 2024-11-13T05:50:54,502 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b690e14b23b7431584e7e74578a7595d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731477052113 2024-11-13T05:50:54,502 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9b71f7c5a1416eb5c5415e18367439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731477052620 2024-11-13T05:50:54,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,516 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:54,518 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#129 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:54,518 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1d4afa228ec5485bad4863d197439f30 is 50, key is test_row_0/C:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:54,522 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111391d05a0610d74e27a245b63210f69cab_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391d05a0610d74e27a245b63210f69cab_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:54,523 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/e2381b9c8d534922ac275090078eb193, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:54,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/e2381b9c8d534922ac275090078eb193 is 175, key is test_row_0/A:col10/1731477053963/Put/seqid=0 2024-11-13T05:50:54,527 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#126 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:54,528 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/8bea3cb1c57c44a6b2eecb15b47e5218 is 175, key is test_row_0/A:col10/1731477052620/Put/seqid=0 2024-11-13T05:50:54,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741973_1149 (size=12139) 2024-11-13T05:50:54,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741974_1150 (size=48139) 2024-11-13T05:50:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:54,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741975_1151 (size=31093) 2024-11-13T05:50:54,569 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/8bea3cb1c57c44a6b2eecb15b47e5218 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/8bea3cb1c57c44a6b2eecb15b47e5218 2024-11-13T05:50:54,579 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into 8bea3cb1c57c44a6b2eecb15b47e5218(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:54,579 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:54,579 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=12, startTime=1731477053953; duration=0sec 2024-11-13T05:50:54,579 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:54,579 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:50:54,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477114791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477114791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477114801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477114802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:54,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:54,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/e2381b9c8d534922ac275090078eb193 2024-11-13T05:50:54,946 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1d4afa228ec5485bad4863d197439f30 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1d4afa228ec5485bad4863d197439f30 2024-11-13T05:50:54,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/b7754855660742cb84a2eb38bb405d1b is 50, key is test_row_0/B:col10/1731477053963/Put/seqid=0 2024-11-13T05:50:54,953 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into 1d4afa228ec5485bad4863d197439f30(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:54,953 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:54,953 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=12, startTime=1731477053960; duration=0sec 2024-11-13T05:50:54,953 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:54,953 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:50:54,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741976_1152 (size=12001) 2024-11-13T05:50:54,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:54,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:54,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:55,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:55,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:55,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477115301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477115301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:55,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477115312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477115312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:55,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/b7754855660742cb84a2eb38bb405d1b 2024-11-13T05:50:55,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/27fbe7472f2c4dfd843dd8ba3603d9f8 is 50, key is test_row_0/C:col10/1731477053963/Put/seqid=0 2024-11-13T05:50:55,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741977_1153 (size=12001) 2024-11-13T05:50:55,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/27fbe7472f2c4dfd843dd8ba3603d9f8 2024-11-13T05:50:55,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/e2381b9c8d534922ac275090078eb193 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193 2024-11-13T05:50:55,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193, entries=250, sequenceid=120, filesize=47.0 K 2024-11-13T05:50:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/b7754855660742cb84a2eb38bb405d1b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/b7754855660742cb84a2eb38bb405d1b 2024-11-13T05:50:55,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/b7754855660742cb84a2eb38bb405d1b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-13T05:50:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/27fbe7472f2c4dfd843dd8ba3603d9f8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/27fbe7472f2c4dfd843dd8ba3603d9f8 2024-11-13T05:50:55,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:55,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/27fbe7472f2c4dfd843dd8ba3603d9f8, entries=150, sequenceid=120, filesize=11.7 K 2024-11-13T05:50:55,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.95 KB/128976, currentSize=147.60 KB/151140 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1467ms, sequenceid=120, compaction requested=false 2024-11-13T05:50:55,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:55,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T05:50:55,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:55,590 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-13T05:50:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:55,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a077b8b1a4b9447c99ea94b923dcfaeb_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477054148/Put/seqid=0 2024-11-13T05:50:55,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741978_1154 (size=12304) 2024-11-13T05:50:55,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:55,630 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a077b8b1a4b9447c99ea94b923dcfaeb_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a077b8b1a4b9447c99ea94b923dcfaeb_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:55,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/40ca86318ede437493496eb2188108c4, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/40ca86318ede437493496eb2188108c4 is 175, key is test_row_0/A:col10/1731477054148/Put/seqid=0 2024-11-13T05:50:55,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741979_1155 (size=31105) 2024-11-13T05:50:55,649 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/40ca86318ede437493496eb2188108c4 2024-11-13T05:50:55,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/3e5cb7f8969f4a79a72d41b221c6ce36 is 50, key is test_row_0/B:col10/1731477054148/Put/seqid=0 2024-11-13T05:50:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:55,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:55,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741980_1156 (size=12151) 2024-11-13T05:50:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477115706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:55,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477115821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:55,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:55,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477116031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,088 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/3e5cb7f8969f4a79a72d41b221c6ce36 2024-11-13T05:50:56,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3a4e56b73daa49bdb73b4f4c827b6bab is 50, key is test_row_0/C:col10/1731477054148/Put/seqid=0 2024-11-13T05:50:56,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741981_1157 (size=12151) 2024-11-13T05:50:56,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477116322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:56,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477116322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:56,330 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,332 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477116334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:56,342 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477116334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:56,344 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477116344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:56,504 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3a4e56b73daa49bdb73b4f4c827b6bab 2024-11-13T05:50:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/40ca86318ede437493496eb2188108c4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4 2024-11-13T05:50:56,519 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4, entries=150, sequenceid=160, filesize=30.4 K 2024-11-13T05:50:56,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/3e5cb7f8969f4a79a72d41b221c6ce36 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3e5cb7f8969f4a79a72d41b221c6ce36 2024-11-13T05:50:56,536 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3e5cb7f8969f4a79a72d41b221c6ce36, entries=150, sequenceid=160, filesize=11.9 K 2024-11-13T05:50:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3a4e56b73daa49bdb73b4f4c827b6bab as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a4e56b73daa49bdb73b4f4c827b6bab 2024-11-13T05:50:56,544 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a4e56b73daa49bdb73b4f4c827b6bab, entries=150, sequenceid=160, filesize=11.9 K 2024-11-13T05:50:56,545 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 956ms, sequenceid=160, compaction requested=true 2024-11-13T05:50:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T05:50:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T05:50:56,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T05:50:56,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1420 sec 2024-11-13T05:50:56,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.1510 sec 2024-11-13T05:50:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:56,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:50:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:56,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:56,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113de6658d6f0e44e528d1cbead502d104d_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:56,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741982_1158 (size=14794) 2024-11-13T05:50:56,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477056936, max timestamp=1731477056966Num uniques 859; 2024-11-13T05:50:56,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477056936, max timestamp=1731477056966Num uniques 859; 2024-11-13T05:50:56,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477056936, max timestamp=1731477056966Num uniques 859; 2024-11-13T05:50:56,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:56,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:56,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:57,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477117060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477117171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:57,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:57,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113de6658d6f0e44e528d1cbead502d104d_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de6658d6f0e44e528d1cbead502d104d_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:57,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/256343c001ce404cb47be0a7b3d0c0f4, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:57,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/256343c001ce404cb47be0a7b3d0c0f4 is 175, key is test_row_0/A:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:57,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741983_1159 (size=39749) 2024-11-13T05:50:57,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477117381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T05:50:57,573 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:57,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T05:50:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:50:57,577 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:57,577 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:57,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:50:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477117691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:57,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/256343c001ce404cb47be0a7b3d0c0f4 2024-11-13T05:50:57,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/3f7cd668866e47f8ae5209b8c5346e9f is 50, key is test_row_0/B:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:57,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741984_1160 (size=12151) 2024-11-13T05:50:57,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/3f7cd668866e47f8ae5209b8c5346e9f 2024-11-13T05:50:57,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:50:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:50:57,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ddeec72a355e47d4af6f31bb0d0c7b30 is 50, key is test_row_0/C:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:57,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741985_1161 (size=12151) 2024-11-13T05:50:57,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ddeec72a355e47d4af6f31bb0d0c7b30 2024-11-13T05:50:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/256343c001ce404cb47be0a7b3d0c0f4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4 2024-11-13T05:50:57,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4, entries=200, sequenceid=170, filesize=38.8 K 2024-11-13T05:50:57,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/3f7cd668866e47f8ae5209b8c5346e9f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3f7cd668866e47f8ae5209b8c5346e9f 2024-11-13T05:50:57,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3f7cd668866e47f8ae5209b8c5346e9f, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T05:50:57,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ddeec72a355e47d4af6f31bb0d0c7b30 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ddeec72a355e47d4af6f31bb0d0c7b30 2024-11-13T05:50:57,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ddeec72a355e47d4af6f31bb0d0c7b30, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T05:50:57,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 924ms, sequenceid=170, compaction requested=true 2024-11-13T05:50:57,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:57,791 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:57,794 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150086 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:57,794 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:50:57,794 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,794 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/8bea3cb1c57c44a6b2eecb15b47e5218, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=146.6 K 2024-11-13T05:50:57,794 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,794 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/8bea3cb1c57c44a6b2eecb15b47e5218, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4] 2024-11-13T05:50:57,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:50:57,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:57,795 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bea3cb1c57c44a6b2eecb15b47e5218, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731477052620 2024-11-13T05:50:57,795 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2381b9c8d534922ac275090078eb193, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731477053300 2024-11-13T05:50:57,795 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:50:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:50:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:57,799 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:57,799 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ca86318ede437493496eb2188108c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731477054148 2024-11-13T05:50:57,799 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:50:57,799 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,799 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5d5f5986213d4d779dbf3434575d0c08, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/b7754855660742cb84a2eb38bb405d1b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3e5cb7f8969f4a79a72d41b221c6ce36, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3f7cd668866e47f8ae5209b8c5346e9f] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=47.3 K 2024-11-13T05:50:57,802 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 256343c001ce404cb47be0a7b3d0c0f4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731477055693 2024-11-13T05:50:57,802 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d5f5986213d4d779dbf3434575d0c08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731477052620 2024-11-13T05:50:57,803 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b7754855660742cb84a2eb38bb405d1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731477053957 2024-11-13T05:50:57,803 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e5cb7f8969f4a79a72d41b221c6ce36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731477054148 2024-11-13T05:50:57,803 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f7cd668866e47f8ae5209b8c5346e9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731477055693 2024-11-13T05:50:57,826 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:57,826 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/f94accc162ce4eff9c8ccc1b13c93adb is 50, key is test_row_0/B:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:57,839 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:57,861 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113febeddbebb5e4f99831941f43ef67799_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:57,865 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113febeddbebb5e4f99831941f43ef67799_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:57,866 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113febeddbebb5e4f99831941f43ef67799_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:57,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741986_1162 (size=12425) 2024-11-13T05:50:57,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T05:50:57,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,886 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T05:50:57,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:57,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:50:57,892 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/f94accc162ce4eff9c8ccc1b13c93adb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f94accc162ce4eff9c8ccc1b13c93adb 2024-11-13T05:50:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:50:57,902 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into f94accc162ce4eff9c8ccc1b13c93adb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:57,902 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:57,902 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=12, startTime=1731477057795; duration=0sec 2024-11-13T05:50:57,902 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:50:57,902 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:50:57,902 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:50:57,904 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:50:57,904 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:50:57,904 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:57,904 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1d4afa228ec5485bad4863d197439f30, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/27fbe7472f2c4dfd843dd8ba3603d9f8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a4e56b73daa49bdb73b4f4c827b6bab, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ddeec72a355e47d4af6f31bb0d0c7b30] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=47.3 K 2024-11-13T05:50:57,905 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4afa228ec5485bad4863d197439f30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731477052620 2024-11-13T05:50:57,905 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fbe7472f2c4dfd843dd8ba3603d9f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731477053957 2024-11-13T05:50:57,905 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4e56b73daa49bdb73b4f4c827b6bab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731477054148 2024-11-13T05:50:57,906 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ddeec72a355e47d4af6f31bb0d0c7b30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731477055693 2024-11-13T05:50:57,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741987_1163 (size=4469) 2024-11-13T05:50:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111306b2b6d85e004d1db6360da9d2a41ed2_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477057005/Put/seqid=0 2024-11-13T05:50:57,942 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#139 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:57,943 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/18d14ee7d19448f08bcf1f37631cf382 is 175, key is test_row_0/A:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:57,943 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:50:57,944 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3adcc9a742d04ef48a3277e8ef6e2690 is 50, key is test_row_0/C:col10/1731477055693/Put/seqid=0 2024-11-13T05:50:58,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741988_1164 (size=12304) 2024-11-13T05:50:58,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:58,008 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111306b2b6d85e004d1db6360da9d2a41ed2_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111306b2b6d85e004d1db6360da9d2a41ed2_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:58,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/03bb84288821482085db82cc1d43b722, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:58,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/03bb84288821482085db82cc1d43b722 is 175, key is test_row_0/A:col10/1731477057005/Put/seqid=0 2024-11-13T05:50:58,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741989_1165 (size=31379) 2024-11-13T05:50:58,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741990_1166 (size=12425) 2024-11-13T05:50:58,049 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3adcc9a742d04ef48a3277e8ef6e2690 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3adcc9a742d04ef48a3277e8ef6e2690 2024-11-13T05:50:58,056 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into 3adcc9a742d04ef48a3277e8ef6e2690(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:58,056 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:58,056 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=12, startTime=1731477057796; duration=0sec 2024-11-13T05:50:58,056 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:58,056 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:50:58,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741991_1167 (size=31105) 2024-11-13T05:50:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:50:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:58,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477118256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477118342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,346 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477118352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,354 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477118361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,369 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477118371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477118372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,379 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:50:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,432 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/18d14ee7d19448f08bcf1f37631cf382 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/18d14ee7d19448f08bcf1f37631cf382 2024-11-13T05:50:58,440 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into 18d14ee7d19448f08bcf1f37631cf382(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:50:58,440 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:58,440 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=12, startTime=1731477057791; duration=0sec 2024-11-13T05:50:58,440 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:50:58,440 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:50:58,469 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/03bb84288821482085db82cc1d43b722 2024-11-13T05:50:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/7828b7e107eb43bfa27ea19638b809b6 is 50, key is test_row_0/B:col10/1731477057005/Put/seqid=0 2024-11-13T05:50:58,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741992_1168 (size=12151) 2024-11-13T05:50:58,525 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/7828b7e107eb43bfa27ea19638b809b6 2024-11-13T05:50:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1a84b4ff9dba4a3daf414ba99930e63d is 50, key is test_row_0/C:col10/1731477057005/Put/seqid=0 2024-11-13T05:50:58,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741993_1169 (size=12151) 2024-11-13T05:50:58,560 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1a84b4ff9dba4a3daf414ba99930e63d 2024-11-13T05:50:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/03bb84288821482085db82cc1d43b722 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722 2024-11-13T05:50:58,573 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722, entries=150, sequenceid=207, filesize=30.4 K 2024-11-13T05:50:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/7828b7e107eb43bfa27ea19638b809b6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/7828b7e107eb43bfa27ea19638b809b6 2024-11-13T05:50:58,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477118581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:58,582 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/7828b7e107eb43bfa27ea19638b809b6, entries=150, sequenceid=207, filesize=11.9 K 2024-11-13T05:50:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:58,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1a84b4ff9dba4a3daf414ba99930e63d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1a84b4ff9dba4a3daf414ba99930e63d 2024-11-13T05:50:58,591 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1a84b4ff9dba4a3daf414ba99930e63d, entries=150, sequenceid=207, filesize=11.9 K 2024-11-13T05:50:58,593 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 706ms, sequenceid=207, compaction requested=false 2024-11-13T05:50:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T05:50:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T05:50:58,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T05:50:58,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-13T05:50:58,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0250 sec 2024-11-13T05:50:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T05:50:58,712 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:50:58,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:50:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T05:50:58,715 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:50:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:50:58,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:50:58,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:50:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:50:58,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T05:50:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:58,871 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:50:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:50:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:50:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:50:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:50:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132e24f623bc6941ed8b26c9c2ec1a97a6_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477058228/Put/seqid=0 2024-11-13T05:50:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:50:58,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741994_1170 (size=12304) 2024-11-13T05:50:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:50:59,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477058936, max timestamp=1731477059056Num uniques 859; 2024-11-13T05:50:59,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:59,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477058936, max timestamp=1731477059056Num uniques 859; 2024-11-13T05:50:59,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477058936, max timestamp=1731477059056Num uniques 859; 2024-11-13T05:50:59,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:50:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477119175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477119282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:59,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:50:59,323 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132e24f623bc6941ed8b26c9c2ec1a97a6_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132e24f623bc6941ed8b26c9c2ec1a97a6_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:50:59,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/974a17a4fc884808b2066cd4bcb593a5, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:50:59,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/974a17a4fc884808b2066cd4bcb593a5 is 175, key is test_row_0/A:col10/1731477058228/Put/seqid=0 2024-11-13T05:50:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:50:59,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741995_1171 (size=31105) 2024-11-13T05:50:59,353 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/974a17a4fc884808b2066cd4bcb593a5 2024-11-13T05:50:59,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/6ed92cca4bca4c6d96fc595f42ff0320 is 50, key is test_row_0/B:col10/1731477058228/Put/seqid=0 2024-11-13T05:50:59,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741996_1172 (size=12151) 2024-11-13T05:50:59,397 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/6ed92cca4bca4c6d96fc595f42ff0320 2024-11-13T05:50:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/b1b0bd810ef54f33a8a8d9c8feab1811 is 50, key is test_row_0/C:col10/1731477058228/Put/seqid=0 2024-11-13T05:50:59,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741997_1173 (size=12151) 2024-11-13T05:50:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477119501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:50:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477119812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:50:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:50:59,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:50:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:50:59,869 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/b1b0bd810ef54f33a8a8d9c8feab1811 2024-11-13T05:50:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/974a17a4fc884808b2066cd4bcb593a5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5 2024-11-13T05:50:59,895 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5, entries=150, sequenceid=220, filesize=30.4 K 2024-11-13T05:50:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/6ed92cca4bca4c6d96fc595f42ff0320 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6ed92cca4bca4c6d96fc595f42ff0320 2024-11-13T05:50:59,901 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6ed92cca4bca4c6d96fc595f42ff0320, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T05:50:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/b1b0bd810ef54f33a8a8d9c8feab1811 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b1b0bd810ef54f33a8a8d9c8feab1811 2024-11-13T05:50:59,910 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b1b0bd810ef54f33a8a8d9c8feab1811, entries=150, sequenceid=220, filesize=11.9 K 2024-11-13T05:50:59,911 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1040ms, sequenceid=220, compaction requested=true 2024-11-13T05:50:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:50:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:50:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T05:50:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T05:50:59,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T05:50:59,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-13T05:50:59,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2020 sec 2024-11-13T05:51:00,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477059077, max timestamp=1731477060330Num uniques 1018; 2024-11-13T05:51:00,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477059077, max timestamp=1731477060330Num uniques 1018; 2024-11-13T05:51:00,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T05:51:00,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477059077, max timestamp=1731477060330Num uniques 1018; 2024-11-13T05:51:00,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T05:51:00,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T05:51:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:00,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-13T05:51:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:00,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111382fdaafdd2a34ff6af6c222a743a33ad_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741998_1174 (size=14994) 2024-11-13T05:51:00,395 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:00,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111382fdaafdd2a34ff6af6c222a743a33ad_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111382fdaafdd2a34ff6af6c222a743a33ad_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:00,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/61c0954fe3e04633a7371ee7e92f7cb1, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:00,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/61c0954fe3e04633a7371ee7e92f7cb1 is 175, key is test_row_0/A:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741999_1175 (size=39949) 2024-11-13T05:51:00,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477060420, max timestamp=1731477060496Num uniques 859; 2024-11-13T05:51:00,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:00,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477060420, max timestamp=1731477060496Num uniques 859; 2024-11-13T05:51:00,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477060420, max timestamp=1731477060496Num uniques 859; 2024-11-13T05:51:00,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:00,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:00,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477120627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477120741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:00,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/61c0954fe3e04633a7371ee7e92f7cb1 2024-11-13T05:51:00,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/0aec00c01c0f4bb48d5a605ca55c9470 is 50, key is test_row_0/B:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T05:51:00,862 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:00,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-13T05:51:00,865 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T05:51:00,866 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:00,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:00,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742000_1176 (size=12301) 2024-11-13T05:51:00,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/0aec00c01c0f4bb48d5a605ca55c9470 2024-11-13T05:51:00,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/58cf8051aefc49629b115d63e27262e6 is 50, key is test_row_0/C:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:00,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742001_1177 (size=12301) 2024-11-13T05:51:00,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477120951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T05:51:01,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T05:51:01,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:01,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T05:51:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:01,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T05:51:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477121271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:01,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T05:51:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:01,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:01,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/58cf8051aefc49629b115d63e27262e6 2024-11-13T05:51:01,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/61c0954fe3e04633a7371ee7e92f7cb1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1 2024-11-13T05:51:01,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1, entries=200, sequenceid=262, filesize=39.0 K 2024-11-13T05:51:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/0aec00c01c0f4bb48d5a605ca55c9470 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/0aec00c01c0f4bb48d5a605ca55c9470 2024-11-13T05:51:01,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/0aec00c01c0f4bb48d5a605ca55c9470, entries=150, sequenceid=262, filesize=12.0 K 2024-11-13T05:51:01,373 INFO [master/bc386b7bf2fe:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T05:51:01,373 INFO [master/bc386b7bf2fe:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T05:51:01,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/58cf8051aefc49629b115d63e27262e6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/58cf8051aefc49629b115d63e27262e6 2024-11-13T05:51:01,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/58cf8051aefc49629b115d63e27262e6, entries=150, sequenceid=262, filesize=12.0 K 2024-11-13T05:51:01,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1053ms, sequenceid=262, compaction requested=true 2024-11-13T05:51:01,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:01,402 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:01,403 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:01,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:01,422 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133538 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:01,422 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:51:01,422 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,422 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/18d14ee7d19448f08bcf1f37631cf382, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=130.4 K 2024-11-13T05:51:01,422 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,424 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/18d14ee7d19448f08bcf1f37631cf382, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1] 2024-11-13T05:51:01,424 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:01,425 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:51:01,425 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,425 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f94accc162ce4eff9c8ccc1b13c93adb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/7828b7e107eb43bfa27ea19638b809b6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6ed92cca4bca4c6d96fc595f42ff0320, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/0aec00c01c0f4bb48d5a605ca55c9470] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=47.9 K 2024-11-13T05:51:01,425 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f94accc162ce4eff9c8ccc1b13c93adb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731477055693 2024-11-13T05:51:01,426 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18d14ee7d19448f08bcf1f37631cf382, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731477055693 2024-11-13T05:51:01,426 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7828b7e107eb43bfa27ea19638b809b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731477057005 2024-11-13T05:51:01,426 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03bb84288821482085db82cc1d43b722, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731477057005 2024-11-13T05:51:01,427 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed92cca4bca4c6d96fc595f42ff0320, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731477058228 2024-11-13T05:51:01,427 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974a17a4fc884808b2066cd4bcb593a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731477058228 2024-11-13T05:51:01,427 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aec00c01c0f4bb48d5a605ca55c9470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731477060338 2024-11-13T05:51:01,428 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c0954fe3e04633a7371ee7e92f7cb1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731477060335 2024-11-13T05:51:01,472 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:01,480 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:01,480 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/8539d6fd74c0420898a1ad7dfb96753e is 50, key is test_row_0/B:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:01,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,488 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113badbc0f723e24859b404368f464a9c54_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:01,488 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T05:51:01,492 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113badbc0f723e24859b404368f464a9c54_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:01,492 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113badbc0f723e24859b404368f464a9c54_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:01,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e3eca15c2cf94aaa9f4504165105f54e_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477060623/Put/seqid=0 2024-11-13T05:51:01,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742002_1178 (size=12711) 2024-11-13T05:51:01,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742003_1179 (size=4469) 2024-11-13T05:51:01,579 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/8539d6fd74c0420898a1ad7dfb96753e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/8539d6fd74c0420898a1ad7dfb96753e 2024-11-13T05:51:01,579 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#150 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:01,580 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/6563e3bcef9c4fda82b8d659171953cf is 175, key is test_row_0/A:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742004_1180 (size=12454) 2024-11-13T05:51:01,590 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into 8539d6fd74c0420898a1ad7dfb96753e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:01,590 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:01,590 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=12, startTime=1731477061403; duration=0sec 2024-11-13T05:51:01,590 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:01,590 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:51:01,590 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:01,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:01,593 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:01,593 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:51:01,593 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:01,593 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3adcc9a742d04ef48a3277e8ef6e2690, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1a84b4ff9dba4a3daf414ba99930e63d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b1b0bd810ef54f33a8a8d9c8feab1811, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/58cf8051aefc49629b115d63e27262e6] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=47.9 K 2024-11-13T05:51:01,594 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adcc9a742d04ef48a3277e8ef6e2690, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731477055693 2024-11-13T05:51:01,595 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a84b4ff9dba4a3daf414ba99930e63d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731477057005 2024-11-13T05:51:01,597 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b0bd810ef54f33a8a8d9c8feab1811, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731477058228 2024-11-13T05:51:01,597 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cf8051aefc49629b115d63e27262e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731477060338 2024-11-13T05:51:01,599 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e3eca15c2cf94aaa9f4504165105f54e_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e3eca15c2cf94aaa9f4504165105f54e_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/55b46bc1a4ba4a3cb548f0ca1a186bf8, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/55b46bc1a4ba4a3cb548f0ca1a186bf8 is 175, key is test_row_0/A:col10/1731477060623/Put/seqid=0 2024-11-13T05:51:01,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742005_1181 (size=31665) 2024-11-13T05:51:01,634 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/6563e3bcef9c4fda82b8d659171953cf as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/6563e3bcef9c4fda82b8d659171953cf 2024-11-13T05:51:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742006_1182 (size=31255) 2024-11-13T05:51:01,651 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into 6563e3bcef9c4fda82b8d659171953cf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:01,651 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:01,651 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=12, startTime=1731477061402; duration=0sec 2024-11-13T05:51:01,651 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:01,651 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:51:01,675 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:01,676 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3bc691a20ebe497497fac5ba633fff56 is 50, key is test_row_0/C:col10/1731477060338/Put/seqid=0 2024-11-13T05:51:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742007_1183 (size=12711) 2024-11-13T05:51:01,751 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3bc691a20ebe497497fac5ba633fff56 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3bc691a20ebe497497fac5ba633fff56 2024-11-13T05:51:01,757 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into 3bc691a20ebe497497fac5ba633fff56(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:01,757 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:01,757 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=12, startTime=1731477061403; duration=0sec 2024-11-13T05:51:01,757 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:01,757 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:51:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:01,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:01,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477121851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477121962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T05:51:02,049 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/55b46bc1a4ba4a3cb548f0ca1a186bf8 2024-11-13T05:51:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/2a583ee0e2c54f7db15ff8bac9afdc13 is 50, key is test_row_0/B:col10/1731477060623/Put/seqid=0 2024-11-13T05:51:02,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742008_1184 (size=12301) 2024-11-13T05:51:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477122171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59828 deadline: 1731477122371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:02,378 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59810 deadline: 1731477122381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:02,383 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59844 deadline: 1731477122402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:02,403 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59814 deadline: 1731477122421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:02,424 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 5eaaa9d25e2cd5c3d12b429e77fa393b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,476 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/2a583ee0e2c54f7db15ff8bac9afdc13 2024-11-13T05:51:02,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477122482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ab25390d09834ac38b9ec15c7c011c3c is 50, key is test_row_0/C:col10/1731477060623/Put/seqid=0 2024-11-13T05:51:02,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742009_1185 (size=12301) 2024-11-13T05:51:02,893 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ab25390d09834ac38b9ec15c7c011c3c 2024-11-13T05:51:02,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/55b46bc1a4ba4a3cb548f0ca1a186bf8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8 2024-11-13T05:51:02,905 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8, entries=150, sequenceid=298, filesize=30.5 K 2024-11-13T05:51:02,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/2a583ee0e2c54f7db15ff8bac9afdc13 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/2a583ee0e2c54f7db15ff8bac9afdc13 2024-11-13T05:51:02,914 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/2a583ee0e2c54f7db15ff8bac9afdc13, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:51:02,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ab25390d09834ac38b9ec15c7c011c3c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab25390d09834ac38b9ec15c7c011c3c 2024-11-13T05:51:02,925 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab25390d09834ac38b9ec15c7c011c3c, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:51:02,926 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1438ms, sequenceid=298, compaction requested=false 2024-11-13T05:51:02,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:02,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:02,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T05:51:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T05:51:02,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T05:51:02,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0620 sec 2024-11-13T05:51:02,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.0670 sec 2024-11-13T05:51:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:02,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T05:51:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:03,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137b7af7a501664b0a881df1f6d6f466a8_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T05:51:03,012 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:03,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-13T05:51:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:51:03,017 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:03,021 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:03,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:03,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742010_1186 (size=14994) 2024-11-13T05:51:03,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477063056, max timestamp=1731477063098Num uniques 859; 2024-11-13T05:51:03,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477063056, max timestamp=1731477063098Num uniques 859; 2024-11-13T05:51:03,101 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477063056, max timestamp=1731477063098Num uniques 859; 2024-11-13T05:51:03,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:03,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:03,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:51:03,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477123217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:03,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:51:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477123332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:03,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:03,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137b7af7a501664b0a881df1f6d6f466a8_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b7af7a501664b0a881df1f6d6f466a8_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:03,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/85d1652015f6443ca7db3a5a94b3a6b3, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:03,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/85d1652015f6443ca7db3a5a94b3a6b3 is 175, key is test_row_0/A:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:03,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742011_1187 (size=39949) 2024-11-13T05:51:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477123541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:51:03,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477123853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:03,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/85d1652015f6443ca7db3a5a94b3a6b3 2024-11-13T05:51:03,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5b352ef7a87b4bb28c7bc49ac9ce6097 is 50, key is test_row_0/B:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:03,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742012_1188 (size=12301) 2024-11-13T05:51:03,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5b352ef7a87b4bb28c7bc49ac9ce6097 2024-11-13T05:51:03,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/4fad93b6d79a45baabe99addecc38e06 is 50, key is test_row_0/C:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:03,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:03,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:03,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742013_1189 (size=12301) 2024-11-13T05:51:04,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:51:04,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/4fad93b6d79a45baabe99addecc38e06 2024-11-13T05:51:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477124371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:04,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/85d1652015f6443ca7db3a5a94b3a6b3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3 2024-11-13T05:51:04,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3, entries=200, sequenceid=312, filesize=39.0 K 2024-11-13T05:51:04,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5b352ef7a87b4bb28c7bc49ac9ce6097 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5b352ef7a87b4bb28c7bc49ac9ce6097 2024-11-13T05:51:04,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5b352ef7a87b4bb28c7bc49ac9ce6097, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T05:51:04,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/4fad93b6d79a45baabe99addecc38e06 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/4fad93b6d79a45baabe99addecc38e06 2024-11-13T05:51:04,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:04,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/4fad93b6d79a45baabe99addecc38e06, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T05:51:04,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1420ms, sequenceid=312, compaction requested=true 2024-11-13T05:51:04,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:04,414 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:04,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:51:04,414 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:04,419 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:04,419 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:51:04,419 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,419 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:04,419 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:51:04,419 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,419 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/6563e3bcef9c4fda82b8d659171953cf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=100.5 K 2024-11-13T05:51:04,419 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/8539d6fd74c0420898a1ad7dfb96753e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/2a583ee0e2c54f7db15ff8bac9afdc13, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5b352ef7a87b4bb28c7bc49ac9ce6097] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=36.4 K 2024-11-13T05:51:04,419 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,419 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/6563e3bcef9c4fda82b8d659171953cf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3] 2024-11-13T05:51:04,420 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8539d6fd74c0420898a1ad7dfb96753e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731477060338 2024-11-13T05:51:04,420 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a583ee0e2c54f7db15ff8bac9afdc13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477060599 2024-11-13T05:51:04,420 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6563e3bcef9c4fda82b8d659171953cf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731477060338 2024-11-13T05:51:04,422 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b46bc1a4ba4a3cb548f0ca1a186bf8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477060599 2024-11-13T05:51:04,422 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b352ef7a87b4bb28c7bc49ac9ce6097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731477061829 2024-11-13T05:51:04,423 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d1652015f6443ca7db3a5a94b3a6b3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731477061829 2024-11-13T05:51:04,429 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:04,431 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:04,431 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/4a8a16d43f19449d8d176acb72107194 is 50, key is test_row_0/B:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:04,434 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139d51936578ff43899f2879f9b176b670_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:04,435 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139d51936578ff43899f2879f9b176b670_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:04,436 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d51936578ff43899f2879f9b176b670_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:04,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742014_1190 (size=12813) 2024-11-13T05:51:04,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742015_1191 (size=4469) 2024-11-13T05:51:04,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T05:51:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,562 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-13T05:51:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:04,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343ec765f145f4aeeb7c9a85dcec9b20d_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477063172/Put/seqid=0 2024-11-13T05:51:04,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742016_1192 (size=12454) 2024-11-13T05:51:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:04,588 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343ec765f145f4aeeb7c9a85dcec9b20d_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343ec765f145f4aeeb7c9a85dcec9b20d_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:04,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/5529de4e345d420698c80ce3a8fb807b, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:04,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/5529de4e345d420698c80ce3a8fb807b is 175, key is test_row_0/A:col10/1731477063172/Put/seqid=0 2024-11-13T05:51:04,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742017_1193 (size=31255) 2024-11-13T05:51:04,603 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/5529de4e345d420698c80ce3a8fb807b 2024-11-13T05:51:04,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/08b6fbaf3fe84c6c8583bb080af22232 is 50, key is test_row_0/B:col10/1731477063172/Put/seqid=0 2024-11-13T05:51:04,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742018_1194 (size=12301) 2024-11-13T05:51:04,852 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/4a8a16d43f19449d8d176acb72107194 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/4a8a16d43f19449d8d176acb72107194 2024-11-13T05:51:04,854 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:04,854 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/497179ddeae942fb94a2ae4a8eb3e5d6 is 175, key is test_row_0/A:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:04,857 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into 4a8a16d43f19449d8d176acb72107194(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:04,857 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:04,857 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=13, startTime=1731477064414; duration=0sec 2024-11-13T05:51:04,858 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:04,858 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:51:04,858 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:04,859 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:04,859 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:51:04,859 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:04,859 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3bc691a20ebe497497fac5ba633fff56, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab25390d09834ac38b9ec15c7c011c3c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/4fad93b6d79a45baabe99addecc38e06] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=36.4 K 2024-11-13T05:51:04,859 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc691a20ebe497497fac5ba633fff56, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731477060338 2024-11-13T05:51:04,860 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ab25390d09834ac38b9ec15c7c011c3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477060599 2024-11-13T05:51:04,860 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fad93b6d79a45baabe99addecc38e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731477061829 2024-11-13T05:51:04,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742019_1195 (size=31767) 2024-11-13T05:51:04,868 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:04,869 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1cbdcf0440ea47bf805b60f71d6857e8 is 50, key is test_row_0/C:col10/1731477061836/Put/seqid=0 2024-11-13T05:51:04,869 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/497179ddeae942fb94a2ae4a8eb3e5d6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/497179ddeae942fb94a2ae4a8eb3e5d6 2024-11-13T05:51:04,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742020_1196 (size=12813) 2024-11-13T05:51:04,876 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into 497179ddeae942fb94a2ae4a8eb3e5d6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:04,876 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:04,876 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=13, startTime=1731477064414; duration=0sec 2024-11-13T05:51:04,876 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:04,876 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:51:04,878 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1cbdcf0440ea47bf805b60f71d6857e8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1cbdcf0440ea47bf805b60f71d6857e8 2024-11-13T05:51:04,883 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into 1cbdcf0440ea47bf805b60f71d6857e8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:04,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:04,883 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=13, startTime=1731477064414; duration=0sec 2024-11-13T05:51:04,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:04,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:51:05,020 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/08b6fbaf3fe84c6c8583bb080af22232 2024-11-13T05:51:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/f6982cca69d842e29b0c06e4e8fc32c6 is 50, key is test_row_0/C:col10/1731477063172/Put/seqid=0 2024-11-13T05:51:05,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742021_1197 (size=12301) 2024-11-13T05:51:05,040 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/f6982cca69d842e29b0c06e4e8fc32c6 2024-11-13T05:51:05,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/5529de4e345d420698c80ce3a8fb807b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b 2024-11-13T05:51:05,056 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b, entries=150, sequenceid=346, filesize=30.5 K 2024-11-13T05:51:05,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/08b6fbaf3fe84c6c8583bb080af22232 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/08b6fbaf3fe84c6c8583bb080af22232 2024-11-13T05:51:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,063 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/08b6fbaf3fe84c6c8583bb080af22232, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T05:51:05,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/f6982cca69d842e29b0c06e4e8fc32c6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/f6982cca69d842e29b0c06e4e8fc32c6 2024-11-13T05:51:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,072 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/f6982cca69d842e29b0c06e4e8fc32c6, entries=150, sequenceid=346, filesize=12.0 K 2024-11-13T05:51:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,073 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 511ms, sequenceid=346, compaction requested=false 2024-11-13T05:51:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:05,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-13T05:51:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-13T05:51:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T05:51:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-13T05:51:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0680 sec 2024-11-13T05:51:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T05:51:05,172 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-13T05:51:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T05:51:05,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,177 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T05:51:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T05:51:05,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T05:51:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T05:51:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T05:51:05,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-13T05:51:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 163 msec 2024-11-13T05:51:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T05:51:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,492 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T05:51:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:51:05,496 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,497 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:51:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:51:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:05,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:05,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:05,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:05,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:05,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:51:05,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138159ff99ce7c4e62be9da341363a6771_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:05,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:05,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742022_1198 (size=14994) 2024-11-13T05:51:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,681 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138159ff99ce7c4e62be9da341363a6771_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138159ff99ce7c4e62be9da341363a6771_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/98c401d6590c4c469b02369d8ffd3d47, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/98c401d6590c4c469b02369d8ffd3d47 is 175, key is test_row_0/A:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742023_1199 (size=39945) 2024-11-13T05:51:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/98c401d6590c4c469b02369d8ffd3d47 2024-11-13T05:51:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:05,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5f635957bd6e4b60b8ce7dfbb510bfab is 50, key is test_row_0/B:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:51:05,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:51:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742024_1200 (size=9857) 2024-11-13T05:51:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477065801, max timestamp=1731477065833Num uniques 859; 2024-11-13T05:51:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477065715, max timestamp=1731477065833Num uniques 859; 2024-11-13T05:51:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477065801, max timestamp=1731477065833Num uniques 859; 2024-11-13T05:51:05,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:05,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:51:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:05,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:05,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477125969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477126081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:51:06,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:51:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:06,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5f635957bd6e4b60b8ce7dfbb510bfab 2024-11-13T05:51:06,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/446ce01e29a44be0a04583ed0eeb2b37 is 50, key is test_row_0/C:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742025_1201 (size=9857) 2024-11-13T05:51:06,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/446ce01e29a44be0a04583ed0eeb2b37 2024-11-13T05:51:06,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/98c401d6590c4c469b02369d8ffd3d47 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47 2024-11-13T05:51:06,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:51:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:06,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:06,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47, entries=200, sequenceid=360, filesize=39.0 K 2024-11-13T05:51:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:06,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5f635957bd6e4b60b8ce7dfbb510bfab as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5f635957bd6e4b60b8ce7dfbb510bfab 2024-11-13T05:51:06,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5f635957bd6e4b60b8ce7dfbb510bfab, entries=100, sequenceid=360, filesize=9.6 K 2024-11-13T05:51:06,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/446ce01e29a44be0a04583ed0eeb2b37 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/446ce01e29a44be0a04583ed0eeb2b37 2024-11-13T05:51:06,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477126291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:06,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/446ce01e29a44be0a04583ed0eeb2b37, entries=100, sequenceid=360, filesize=9.6 K 2024-11-13T05:51:06,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 704ms, sequenceid=360, compaction requested=true 2024-11-13T05:51:06,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:06,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:06,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:06,330 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:06,330 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:06,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:06,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:06,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:06,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:06,333 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:06,333 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:51:06,334 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,334 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/4a8a16d43f19449d8d176acb72107194, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/08b6fbaf3fe84c6c8583bb080af22232, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5f635957bd6e4b60b8ce7dfbb510bfab] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=34.2 K 2024-11-13T05:51:06,334 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:06,334 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:51:06,334 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,335 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/497179ddeae942fb94a2ae4a8eb3e5d6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=100.6 K 2024-11-13T05:51:06,335 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,335 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/497179ddeae942fb94a2ae4a8eb3e5d6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47] 2024-11-13T05:51:06,335 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a8a16d43f19449d8d176acb72107194, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731477061829 2024-11-13T05:51:06,335 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b6fbaf3fe84c6c8583bb080af22232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731477063172 2024-11-13T05:51:06,336 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497179ddeae942fb94a2ae4a8eb3e5d6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731477061829 2024-11-13T05:51:06,336 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f635957bd6e4b60b8ce7dfbb510bfab, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731477065537 2024-11-13T05:51:06,337 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5529de4e345d420698c80ce3a8fb807b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731477063172 2024-11-13T05:51:06,337 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c401d6590c4c469b02369d8ffd3d47, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731477065537 2024-11-13T05:51:06,357 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:06,358 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/54f6256e44654458b8c1be88a08ede87 is 50, key is test_row_0/B:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:06,371 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:06,380 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113b607e744719e46fdb1645a1deaebe6a2_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:06,382 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113b607e744719e46fdb1645a1deaebe6a2_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:06,382 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b607e744719e46fdb1645a1deaebe6a2_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:06,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742026_1202 (size=12915) 2024-11-13T05:51:06,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742027_1203 (size=4469) 2024-11-13T05:51:06,399 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/54f6256e44654458b8c1be88a08ede87 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/54f6256e44654458b8c1be88a08ede87 2024-11-13T05:51:06,408 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into 54f6256e44654458b8c1be88a08ede87(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:06,408 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:06,408 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=13, startTime=1731477066330; duration=0sec 2024-11-13T05:51:06,408 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:06,408 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:51:06,408 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:06,411 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:06,411 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:51:06,411 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,411 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1cbdcf0440ea47bf805b60f71d6857e8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/f6982cca69d842e29b0c06e4e8fc32c6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/446ce01e29a44be0a04583ed0eeb2b37] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=34.2 K 2024-11-13T05:51:06,412 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cbdcf0440ea47bf805b60f71d6857e8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731477061829 2024-11-13T05:51:06,412 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f6982cca69d842e29b0c06e4e8fc32c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731477063172 2024-11-13T05:51:06,413 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 446ce01e29a44be0a04583ed0eeb2b37, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731477065537 2024-11-13T05:51:06,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T05:51:06,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,431 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-13T05:51:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:06,438 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:06,438 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ce2744d3cab74701ae4e404790676939 is 50, key is test_row_0/C:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:06,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113510223fc5ce44e639f0d35f7621328df_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477065923/Put/seqid=0 2024-11-13T05:51:06,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742028_1204 (size=12915) 2024-11-13T05:51:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742029_1205 (size=12454) 2024-11-13T05:51:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:06,524 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113510223fc5ce44e639f0d35f7621328df_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113510223fc5ce44e639f0d35f7621328df_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ea65a164e9614d768d53dfbb8ae62eee, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ea65a164e9614d768d53dfbb8ae62eee is 175, key is test_row_0/A:col10/1731477065923/Put/seqid=0 2024-11-13T05:51:06,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742030_1206 (size=31255) 2024-11-13T05:51:06,547 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ea65a164e9614d768d53dfbb8ae62eee 2024-11-13T05:51:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/d17dcd50194040608483d45bc11597f9 is 50, key is test_row_0/B:col10/1731477065923/Put/seqid=0 2024-11-13T05:51:06,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742031_1207 (size=12301) 2024-11-13T05:51:06,585 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/d17dcd50194040608483d45bc11597f9 2024-11-13T05:51:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/5595bf7b22454a15b854bb2fbef62f4b is 50, key is test_row_0/C:col10/1731477065923/Put/seqid=0 2024-11-13T05:51:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:06,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:06,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742032_1208 (size=12301) 2024-11-13T05:51:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:51:06,632 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/5595bf7b22454a15b854bb2fbef62f4b 2024-11-13T05:51:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ea65a164e9614d768d53dfbb8ae62eee as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee 2024-11-13T05:51:06,648 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee, entries=150, sequenceid=398, filesize=30.5 K 2024-11-13T05:51:06,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477126638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/d17dcd50194040608483d45bc11597f9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/d17dcd50194040608483d45bc11597f9 2024-11-13T05:51:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:06,654 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/d17dcd50194040608483d45bc11597f9, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:51:06,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/5595bf7b22454a15b854bb2fbef62f4b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/5595bf7b22454a15b854bb2fbef62f4b 2024-11-13T05:51:06,668 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/5595bf7b22454a15b854bb2fbef62f4b, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T05:51:06,669 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 237ms, sequenceid=398, compaction requested=false 2024-11-13T05:51:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T05:51:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T05:51:06,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T05:51:06,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1730 sec 2024-11-13T05:51:06,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.1810 sec 2024-11-13T05:51:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:06,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T05:51:06,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:06,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:06,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:06,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:06,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:06,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:06,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b77b95f84f184833822e43e20a021932_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:06,794 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:06,795 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/86b03d8f45c14504a9782f7ebd4e24e2 is 175, key is test_row_0/A:col10/1731477065537/Put/seqid=0 2024-11-13T05:51:06,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742033_1209 (size=14994) 2024-11-13T05:51:06,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742034_1210 (size=31976) 2024-11-13T05:51:06,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477066774, max timestamp=1731477066865Num uniques 859; 2024-11-13T05:51:06,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477066774, max timestamp=1731477066865Num uniques 859; 2024-11-13T05:51:06,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:06,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:06,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477066774, max timestamp=1731477066865Num uniques 859; 2024-11-13T05:51:06,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:06,918 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ce2744d3cab74701ae4e404790676939 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ce2744d3cab74701ae4e404790676939 2024-11-13T05:51:06,938 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into ce2744d3cab74701ae4e404790676939(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:06,938 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:06,938 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=13, startTime=1731477066331; duration=0sec 2024-11-13T05:51:06,938 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:06,938 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:51:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477126993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477127101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:07,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:07,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b77b95f84f184833822e43e20a021932_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b77b95f84f184833822e43e20a021932_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:07,222 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/86b03d8f45c14504a9782f7ebd4e24e2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/86b03d8f45c14504a9782f7ebd4e24e2 2024-11-13T05:51:07,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/c267906d7ecf4ff784a26137d9b571a7, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:07,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/c267906d7ecf4ff784a26137d9b571a7 is 175, key is test_row_0/A:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:07,230 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into 86b03d8f45c14504a9782f7ebd4e24e2(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:07,230 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:07,230 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=13, startTime=1731477066330; duration=0sec 2024-11-13T05:51:07,230 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:07,230 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:51:07,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742035_1211 (size=39949) 2024-11-13T05:51:07,253 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/c267906d7ecf4ff784a26137d9b571a7 2024-11-13T05:51:07,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/40b6089035864d718048379af9457e3a is 50, key is test_row_0/B:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:07,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742036_1212 (size=12301) 2024-11-13T05:51:07,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/40b6089035864d718048379af9457e3a 2024-11-13T05:51:07,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/7c9ce20c2b6a44e79920471fc25026fa is 50, key is test_row_0/C:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:07,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477127311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742037_1213 (size=12301) 2024-11-13T05:51:07,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/7c9ce20c2b6a44e79920471fc25026fa 2024-11-13T05:51:07,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/c267906d7ecf4ff784a26137d9b571a7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7 2024-11-13T05:51:07,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7, entries=200, sequenceid=409, filesize=39.0 K 2024-11-13T05:51:07,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/40b6089035864d718048379af9457e3a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/40b6089035864d718048379af9457e3a 2024-11-13T05:51:07,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/40b6089035864d718048379af9457e3a, entries=150, sequenceid=409, filesize=12.0 K 2024-11-13T05:51:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/7c9ce20c2b6a44e79920471fc25026fa as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/7c9ce20c2b6a44e79920471fc25026fa 2024-11-13T05:51:07,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/7c9ce20c2b6a44e79920471fc25026fa, entries=150, sequenceid=409, filesize=12.0 K 2024-11-13T05:51:07,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 603ms, sequenceid=409, compaction requested=true 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:07,367 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:51:07,367 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:07,368 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103180 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:07,368 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:07,368 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:51:07,369 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:51:07,369 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,369 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,369 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/54f6256e44654458b8c1be88a08ede87, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/d17dcd50194040608483d45bc11597f9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/40b6089035864d718048379af9457e3a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=36.6 K 2024-11-13T05:51:07,369 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/86b03d8f45c14504a9782f7ebd4e24e2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=100.8 K 2024-11-13T05:51:07,369 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,369 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/86b03d8f45c14504a9782f7ebd4e24e2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7] 2024-11-13T05:51:07,369 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f6256e44654458b8c1be88a08ede87, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731477063213 2024-11-13T05:51:07,369 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86b03d8f45c14504a9782f7ebd4e24e2, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731477063213 2024-11-13T05:51:07,370 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d17dcd50194040608483d45bc11597f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731477065923 2024-11-13T05:51:07,370 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea65a164e9614d768d53dfbb8ae62eee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731477065923 2024-11-13T05:51:07,371 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c267906d7ecf4ff784a26137d9b571a7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731477066611 2024-11-13T05:51:07,371 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b6089035864d718048379af9457e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731477066611 2024-11-13T05:51:07,391 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:07,405 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:07,405 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/ed08a80c8ec84e4398aa05e3ccfc17b7 is 50, key is test_row_0/B:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:07,423 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138d9cf8ab8b4646aa8ac99279daa405d4_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:07,425 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138d9cf8ab8b4646aa8ac99279daa405d4_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:07,426 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138d9cf8ab8b4646aa8ac99279daa405d4_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:07,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742039_1215 (size=4469) 2024-11-13T05:51:07,441 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#177 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:07,442 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/f2f3b15e85da4d4195167fad80ae570d is 175, key is test_row_0/A:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:07,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742038_1214 (size=13017) 2024-11-13T05:51:07,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742040_1216 (size=31971) 2024-11-13T05:51:07,466 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/f2f3b15e85da4d4195167fad80ae570d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f2f3b15e85da4d4195167fad80ae570d 2024-11-13T05:51:07,472 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into f2f3b15e85da4d4195167fad80ae570d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:07,472 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:07,473 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=13, startTime=1731477067367; duration=0sec 2024-11-13T05:51:07,473 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:07,473 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:51:07,473 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:07,475 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:07,475 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:51:07,475 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,475 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ce2744d3cab74701ae4e404790676939, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/5595bf7b22454a15b854bb2fbef62f4b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/7c9ce20c2b6a44e79920471fc25026fa] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=36.6 K 2024-11-13T05:51:07,476 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2744d3cab74701ae4e404790676939, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731477063213 2024-11-13T05:51:07,476 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5595bf7b22454a15b854bb2fbef62f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731477065923 2024-11-13T05:51:07,477 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9ce20c2b6a44e79920471fc25026fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731477066611 2024-11-13T05:51:07,485 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:07,486 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ab53980e033942f891a084c7b98bd0ef is 50, key is test_row_0/C:col10/1731477066762/Put/seqid=0 2024-11-13T05:51:07,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742041_1217 (size=13017) 2024-11-13T05:51:07,522 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ab53980e033942f891a084c7b98bd0ef as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab53980e033942f891a084c7b98bd0ef 2024-11-13T05:51:07,529 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into ab53980e033942f891a084c7b98bd0ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:07,530 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:07,530 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=13, startTime=1731477067367; duration=0sec 2024-11-13T05:51:07,530 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:07,530 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:51:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:07,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T05:51:07,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:07,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:07,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:07,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:07,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:07,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T05:51:07,642 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:07,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-13T05:51:07,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:07,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:07,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:51:07,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132f237e6579804121933b71f6bfed0595_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477066986/Put/seqid=0 2024-11-13T05:51:07,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477127673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:07,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742042_1218 (size=14994) 2024-11-13T05:51:07,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:07,694 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132f237e6579804121933b71f6bfed0595_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f237e6579804121933b71f6bfed0595_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:07,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/0e56b5ee8aa249f5a5b15a856b072d5b, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:07,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/0e56b5ee8aa249f5a5b15a856b072d5b is 175, key is test_row_0/A:col10/1731477066986/Put/seqid=0 2024-11-13T05:51:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:51:07,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742043_1219 (size=39949) 2024-11-13T05:51:07,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/0e56b5ee8aa249f5a5b15a856b072d5b 2024-11-13T05:51:07,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/6f5a33e455e34dbfbc309a1bafa6b69b is 50, key is test_row_0/B:col10/1731477066986/Put/seqid=0 2024-11-13T05:51:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742044_1220 (size=12301) 2024-11-13T05:51:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477127782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:07,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T05:51:07,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:07,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:07,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:07,854 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/ed08a80c8ec84e4398aa05e3ccfc17b7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/ed08a80c8ec84e4398aa05e3ccfc17b7 2024-11-13T05:51:07,860 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into ed08a80c8ec84e4398aa05e3ccfc17b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:07,860 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:07,860 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=13, startTime=1731477067367; duration=0sec 2024-11-13T05:51:07,860 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:07,860 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:51:07,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T05:51:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:07,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:51:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477127991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:08,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T05:51:08,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:08,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:08,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:08,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:08,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:08,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/6f5a33e455e34dbfbc309a1bafa6b69b 2024-11-13T05:51:08,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/249f2f989c5b4fe0bb2c491eb5f8aaf0 is 50, key is test_row_0/C:col10/1731477066986/Put/seqid=0 2024-11-13T05:51:08,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742045_1221 (size=12301) 2024-11-13T05:51:08,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/249f2f989c5b4fe0bb2c491eb5f8aaf0 2024-11-13T05:51:08,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/0e56b5ee8aa249f5a5b15a856b072d5b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b 2024-11-13T05:51:08,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b, entries=200, sequenceid=450, filesize=39.0 K 2024-11-13T05:51:08,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/6f5a33e455e34dbfbc309a1bafa6b69b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6f5a33e455e34dbfbc309a1bafa6b69b 2024-11-13T05:51:08,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6f5a33e455e34dbfbc309a1bafa6b69b, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T05:51:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/249f2f989c5b4fe0bb2c491eb5f8aaf0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/249f2f989c5b4fe0bb2c491eb5f8aaf0 2024-11-13T05:51:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/249f2f989c5b4fe0bb2c491eb5f8aaf0, entries=150, sequenceid=450, filesize=12.0 K 2024-11-13T05:51:08,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 615ms, sequenceid=450, compaction requested=false 2024-11-13T05:51:08,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-13T05:51:08,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:08,258 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:51:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:51:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136db3f7af0482485c9ebc92bd889619a8_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477067672/Put/seqid=0 2024-11-13T05:51:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742046_1222 (size=9914) 2024-11-13T05:51:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,294 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136db3f7af0482485c9ebc92bd889619a8_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136db3f7af0482485c9ebc92bd889619a8_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:08,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/51731b63f3294fa2850ccc85d921cdfa, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:08,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/51731b63f3294fa2850ccc85d921cdfa is 175, key is test_row_0/A:col10/1731477067672/Put/seqid=0 2024-11-13T05:51:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:08,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742047_1223 (size=22561) 2024-11-13T05:51:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:08,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477068382, max timestamp=1731477068421Num uniques 859; 2024-11-13T05:51:08,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:08,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477068382, max timestamp=1731477068421Num uniques 859; 2024-11-13T05:51:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477068382, max timestamp=1731477068421Num uniques 859; 2024-11-13T05:51:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:08,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:51:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477128501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477128621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:08,719 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/51731b63f3294fa2850ccc85d921cdfa 2024-11-13T05:51:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/e36ddf1cbcb54b4ebd724c4d108c0821 is 50, key is test_row_0/B:col10/1731477067672/Put/seqid=0 2024-11-13T05:51:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742048_1224 (size=9857) 2024-11-13T05:51:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:51:08,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 338 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477128831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:09,143 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/e36ddf1cbcb54b4ebd724c4d108c0821 2024-11-13T05:51:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477129141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:09,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:09,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/8a6e1a3300014ca6939fb48e7fa846c4 is 50, key is test_row_0/C:col10/1731477067672/Put/seqid=0 2024-11-13T05:51:09,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742049_1225 (size=9857) 2024-11-13T05:51:09,555 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/8a6e1a3300014ca6939fb48e7fa846c4 2024-11-13T05:51:09,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/51731b63f3294fa2850ccc85d921cdfa as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa 2024-11-13T05:51:09,564 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa, entries=100, sequenceid=461, filesize=22.0 K 2024-11-13T05:51:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/e36ddf1cbcb54b4ebd724c4d108c0821 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e36ddf1cbcb54b4ebd724c4d108c0821 2024-11-13T05:51:09,570 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e36ddf1cbcb54b4ebd724c4d108c0821, entries=100, sequenceid=461, filesize=9.6 K 2024-11-13T05:51:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/8a6e1a3300014ca6939fb48e7fa846c4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/8a6e1a3300014ca6939fb48e7fa846c4 2024-11-13T05:51:09,578 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/8a6e1a3300014ca6939fb48e7fa846c4, entries=100, sequenceid=461, filesize=9.6 K 2024-11-13T05:51:09,579 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1320ms, sequenceid=461, compaction requested=true 2024-11-13T05:51:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T05:51:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T05:51:09,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T05:51:09,582 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9350 sec 2024-11-13T05:51:09,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9400 sec 2024-11-13T05:51:09,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-13T05:51:09,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:09,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:51:09,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:09,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:51:09,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:09,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:51:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:09,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113adc856147b0444ce9e0446a3ee8cfef5_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477129695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:09,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742050_1226 (size=14994) 2024-11-13T05:51:09,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:09,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113adc856147b0444ce9e0446a3ee8cfef5_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113adc856147b0444ce9e0446a3ee8cfef5_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:09,720 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/f15c0bececd04976b88282ec23419132, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:09,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/f15c0bececd04976b88282ec23419132 is 175, key is test_row_0/A:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:09,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742051_1227 (size=39949) 2024-11-13T05:51:09,728 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=499, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/f15c0bececd04976b88282ec23419132 2024-11-13T05:51:09,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/c07f699c6d0940e687d153b1a5fd2572 is 50, key is test_row_0/B:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:09,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742052_1228 (size=12301) 2024-11-13T05:51:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T05:51:09,791 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:09,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-13T05:51:09,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:09,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:09,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:09,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 348 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477129801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:09,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:51:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:09,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 349 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477130011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:10,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:51:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:10,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/c07f699c6d0940e687d153b1a5fd2572 2024-11-13T05:51:10,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/80cc3e393042450b8b7892c6b427a25f is 50, key is test_row_0/C:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:10,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742053_1229 (size=12301) 2024-11-13T05:51:10,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:51:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 350 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477130331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:10,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:10,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:51:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:10,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:51:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:10,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/80cc3e393042450b8b7892c6b427a25f 2024-11-13T05:51:10,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/f15c0bececd04976b88282ec23419132 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132 2024-11-13T05:51:10,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132, entries=200, sequenceid=499, filesize=39.0 K 2024-11-13T05:51:10,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/c07f699c6d0940e687d153b1a5fd2572 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/c07f699c6d0940e687d153b1a5fd2572 2024-11-13T05:51:10,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/c07f699c6d0940e687d153b1a5fd2572, entries=150, sequenceid=499, filesize=12.0 K 2024-11-13T05:51:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/80cc3e393042450b8b7892c6b427a25f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/80cc3e393042450b8b7892c6b427a25f 2024-11-13T05:51:10,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/80cc3e393042450b8b7892c6b427a25f, entries=150, sequenceid=499, filesize=12.0 K 2024-11-13T05:51:10,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 954ms, sequenceid=499, compaction requested=true 2024-11-13T05:51:10,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:10,608 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:10,609 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:10,610 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:10,610 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:51:10,610 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,610 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f2f3b15e85da4d4195167fad80ae570d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=131.3 K 2024-11-13T05:51:10,610 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,610 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f2f3b15e85da4d4195167fad80ae570d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132] 2024-11-13T05:51:10,611 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f3b15e85da4d4195167fad80ae570d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731477066611 2024-11-13T05:51:10,611 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:10,611 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:51:10,611 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,611 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/ed08a80c8ec84e4398aa05e3ccfc17b7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6f5a33e455e34dbfbc309a1bafa6b69b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e36ddf1cbcb54b4ebd724c4d108c0821, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/c07f699c6d0940e687d153b1a5fd2572] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=46.4 K 2024-11-13T05:51:10,612 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e56b5ee8aa249f5a5b15a856b072d5b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477066977 2024-11-13T05:51:10,612 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ed08a80c8ec84e4398aa05e3ccfc17b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731477066611 2024-11-13T05:51:10,612 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51731b63f3294fa2850ccc85d921cdfa, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731477067663 2024-11-13T05:51:10,613 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5a33e455e34dbfbc309a1bafa6b69b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477066977 2024-11-13T05:51:10,613 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15c0bececd04976b88282ec23419132, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731477068493 2024-11-13T05:51:10,613 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e36ddf1cbcb54b4ebd724c4d108c0821, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731477067663 2024-11-13T05:51:10,613 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting c07f699c6d0940e687d153b1a5fd2572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731477068495 2024-11-13T05:51:10,635 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:10,636 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/e1435fae6383454dae5512fbbb1b6ee8 is 50, key is test_row_0/B:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:10,636 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:10,645 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135c1b6a379bf84aa7bdf0c7574b62b97f_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:10,648 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135c1b6a379bf84aa7bdf0c7574b62b97f_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:10,648 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135c1b6a379bf84aa7bdf0c7574b62b97f_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:10,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742054_1230 (size=13153) 2024-11-13T05:51:10,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742055_1231 (size=4469) 2024-11-13T05:51:10,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-13T05:51:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:10,713 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T05:51:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:10,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113143711942afa4cf6a27097d8ffbd7f5f_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477069692/Put/seqid=0 2024-11-13T05:51:10,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742056_1232 (size=12454) 2024-11-13T05:51:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:10,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. as already flushing 2024-11-13T05:51:10,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477070901, max timestamp=1731477070924Num uniques 859; 2024-11-13T05:51:10,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477070901, max timestamp=1731477070924Num uniques 859; 2024-11-13T05:51:10,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:10,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:10,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477070901, max timestamp=1731477070924Num uniques 859; 2024-11-13T05:51:10,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477131004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:11,055 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:11,055 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/a56c8de73e2c47fb85c04d1f9c888489 is 175, key is test_row_0/A:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:11,057 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/e1435fae6383454dae5512fbbb1b6ee8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e1435fae6383454dae5512fbbb1b6ee8 2024-11-13T05:51:11,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742057_1233 (size=32107) 2024-11-13T05:51:11,062 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into e1435fae6383454dae5512fbbb1b6ee8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:11,063 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:11,063 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=12, startTime=1731477070609; duration=0sec 2024-11-13T05:51:11,063 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:11,063 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:51:11,063 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:11,065 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:11,065 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:51:11,065 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:11,065 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab53980e033942f891a084c7b98bd0ef, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/249f2f989c5b4fe0bb2c491eb5f8aaf0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/8a6e1a3300014ca6939fb48e7fa846c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/80cc3e393042450b8b7892c6b427a25f] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=46.4 K 2024-11-13T05:51:11,066 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ab53980e033942f891a084c7b98bd0ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731477066611 2024-11-13T05:51:11,066 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 249f2f989c5b4fe0bb2c491eb5f8aaf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731477066977 2024-11-13T05:51:11,066 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/a56c8de73e2c47fb85c04d1f9c888489 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/a56c8de73e2c47fb85c04d1f9c888489 2024-11-13T05:51:11,067 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6e1a3300014ca6939fb48e7fa846c4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1731477067663 2024-11-13T05:51:11,067 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cc3e393042450b8b7892c6b427a25f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731477068495 2024-11-13T05:51:11,071 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into a56c8de73e2c47fb85c04d1f9c888489(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:11,071 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:11,071 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=12, startTime=1731477070608; duration=0sec 2024-11-13T05:51:11,071 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:11,071 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:51:11,080 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#192 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:11,080 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ea8b07d0327f47ecb7c61b5ccc0ce807 is 50, key is test_row_0/C:col10/1731477068495/Put/seqid=0 2024-11-13T05:51:11,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742058_1234 (size=13153) 2024-11-13T05:51:11,092 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/ea8b07d0327f47ecb7c61b5ccc0ce807 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ea8b07d0327f47ecb7c61b5ccc0ce807 2024-11-13T05:51:11,099 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into ea8b07d0327f47ecb7c61b5ccc0ce807(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:11,099 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:11,099 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=12, startTime=1731477070609; duration=0sec 2024-11-13T05:51:11,099 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:11,099 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:51:11,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:11,128 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113143711942afa4cf6a27097d8ffbd7f5f_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113143711942afa4cf6a27097d8ffbd7f5f_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:11,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/19444c4f127d486483c080a31b1e73bc, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477131122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/19444c4f127d486483c080a31b1e73bc is 175, key is test_row_0/A:col10/1731477069692/Put/seqid=0 2024-11-13T05:51:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:11,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742059_1235 (size=31255) 2024-11-13T05:51:11,259 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-13T05:51:11,259 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:11,259 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,259 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:11,260 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-13T05:51:11,260 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:11,260 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,260 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:11,261 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-13T05:51:11,261 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:11,261 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,261 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:11,269 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-13T05:51:11,269 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:11,269 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,270 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:11,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:11,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477131342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:11,537 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=508, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/19444c4f127d486483c080a31b1e73bc 2024-11-13T05:51:11,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5489d80f6bbb4ac4abb71bdfb0876335 is 50, key is test_row_0/B:col10/1731477069692/Put/seqid=0 2024-11-13T05:51:11,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742060_1236 (size=12301) 2024-11-13T05:51:11,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 387 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477131651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:11,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:11,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:11,955 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5489d80f6bbb4ac4abb71bdfb0876335 2024-11-13T05:51:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/e17f1d6c0176475da2ac8475bcfff47a is 50, key is test_row_0/C:col10/1731477069692/Put/seqid=0 2024-11-13T05:51:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742061_1237 (size=12301) 2024-11-13T05:51:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 388 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59790 deadline: 1731477132161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:12,367 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/e17f1d6c0176475da2ac8475bcfff47a 2024-11-13T05:51:12,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/19444c4f127d486483c080a31b1e73bc as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc 2024-11-13T05:51:12,377 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc, entries=150, sequenceid=508, filesize=30.5 K 2024-11-13T05:51:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/5489d80f6bbb4ac4abb71bdfb0876335 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5489d80f6bbb4ac4abb71bdfb0876335 2024-11-13T05:51:12,382 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5489d80f6bbb4ac4abb71bdfb0876335, entries=150, sequenceid=508, filesize=12.0 K 2024-11-13T05:51:12,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/e17f1d6c0176475da2ac8475bcfff47a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/e17f1d6c0176475da2ac8475bcfff47a 2024-11-13T05:51:12,388 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/e17f1d6c0176475da2ac8475bcfff47a, entries=150, sequenceid=508, filesize=12.0 K 2024-11-13T05:51:12,389 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1676ms, sequenceid=508, compaction requested=false 2024-11-13T05:51:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T05:51:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T05:51:12,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T05:51:12,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5940 sec 2024-11-13T05:51:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:12,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-13T05:51:12,392 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-13T05:51:12,393 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:12,393 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:12,393 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:12,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.6010 sec 2024-11-13T05:51:12,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113249394841a744ba38fc8cda51ef13740_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_0/A:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742062_1238 (size=12454) 2024-11-13T05:51:12,423 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-13T05:51:12,423 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:12,423 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,423 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:12,483 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-13T05:51:12,483 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:12,483 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,483 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:12,513 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-13T05:51:12,513 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:12,513 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,513 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:12,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:12,806 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:12,810 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113249394841a744ba38fc8cda51ef13740_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113249394841a744ba38fc8cda51ef13740_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:12,811 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/607033f1f2af48a6a276757765769cee, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:12,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/607033f1f2af48a6a276757765769cee is 175, key is test_row_0/A:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:12,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742063_1239 (size=31255) 2024-11-13T05:51:13,183 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-13T05:51:13,183 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:13,183 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:13,183 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:13,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:13,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=548, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/607033f1f2af48a6a276757765769cee 2024-11-13T05:51:13,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/f0908915f2a0467592ca4adfdcbc2bf2 is 50, key is test_row_0/B:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:13,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742064_1240 (size=12301) 2024-11-13T05:51:13,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/f0908915f2a0467592ca4adfdcbc2bf2 2024-11-13T05:51:13,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1f1abc49043740bf9713fcd3bba59385 is 50, key is test_row_0/C:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:13,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742065_1241 (size=12301) 2024-11-13T05:51:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T05:51:13,951 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 332 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2129 2024-11-13T05:51:13,951 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2135 2024-11-13T05:51:13,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:51:13,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 962 2024-11-13T05:51:13,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2885 rows 2024-11-13T05:51:13,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 961 2024-11-13T05:51:13,952 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2883 rows 2024-11-13T05:51:13,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:51:13,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:51:13,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:13,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:13,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:13,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:51:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:13,956 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477073956"}]},"ts":"1731477073956"} 2024-11-13T05:51:13,958 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:51:13,958 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:51:13,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:51:13,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, UNASSIGN}] 2024-11-13T05:51:13,961 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, UNASSIGN 2024-11-13T05:51:13,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:13,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, UNASSIGN because future has completed 2024-11-13T05:51:13,963 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:51:13,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:51:14,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1f1abc49043740bf9713fcd3bba59385 2024-11-13T05:51:14,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/607033f1f2af48a6a276757765769cee as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee 2024-11-13T05:51:14,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee, entries=150, sequenceid=548, filesize=30.5 K 2024-11-13T05:51:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/f0908915f2a0467592ca4adfdcbc2bf2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f0908915f2a0467592ca4adfdcbc2bf2 2024-11-13T05:51:14,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f0908915f2a0467592ca4adfdcbc2bf2, entries=150, sequenceid=548, filesize=12.0 K 2024-11-13T05:51:14,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/1f1abc49043740bf9713fcd3bba59385 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1f1abc49043740bf9713fcd3bba59385 2024-11-13T05:51:14,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1f1abc49043740bf9713fcd3bba59385, entries=150, sequenceid=548, filesize=12.0 K 2024-11-13T05:51:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:14,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=26.84 KB/27480 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1670ms, sequenceid=548, compaction requested=true 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:14,062 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaaa9d25e2cd5c3d12b429e77fa393b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:14,062 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:14,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:14,063 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:14,063 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:14,063 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/A is initiating minor compaction (all files) 2024-11-13T05:51:14,063 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/B is initiating minor compaction (all files) 2024-11-13T05:51:14,063 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/B in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,063 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/A in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,064 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/a56c8de73e2c47fb85c04d1f9c888489, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=92.4 K 2024-11-13T05:51:14,064 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e1435fae6383454dae5512fbbb1b6ee8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5489d80f6bbb4ac4abb71bdfb0876335, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f0908915f2a0467592ca4adfdcbc2bf2] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=36.9 K 2024-11-13T05:51:14,064 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,064 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/a56c8de73e2c47fb85c04d1f9c888489, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee] 2024-11-13T05:51:14,064 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e1435fae6383454dae5512fbbb1b6ee8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731477068495 2024-11-13T05:51:14,064 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56c8de73e2c47fb85c04d1f9c888489, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731477068495 2024-11-13T05:51:14,065 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5489d80f6bbb4ac4abb71bdfb0876335, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731477069680 2024-11-13T05:51:14,065 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19444c4f127d486483c080a31b1e73bc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731477069680 2024-11-13T05:51:14,065 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607033f1f2af48a6a276757765769cee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731477070997 2024-11-13T05:51:14,065 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f0908915f2a0467592ca4adfdcbc2bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731477070997 2024-11-13T05:51:14,077 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:14,078 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#B#compaction#198 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:14,078 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/bd33fdd48cf74c7492d42d2406c32280 is 50, key is test_row_0/B:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:14,082 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bb43831b37d748c1a858f2d6a215b911_5eaaa9d25e2cd5c3d12b429e77fa393b store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:14,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742066_1242 (size=13255) 2024-11-13T05:51:14,104 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/bd33fdd48cf74c7492d42d2406c32280 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/bd33fdd48cf74c7492d42d2406c32280 2024-11-13T05:51:14,111 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/B of 5eaaa9d25e2cd5c3d12b429e77fa393b into bd33fdd48cf74c7492d42d2406c32280(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:14,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:14,111 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/B, priority=13, startTime=1731477074062; duration=0sec 2024-11-13T05:51:14,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:14,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:B 2024-11-13T05:51:14,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:14,112 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:14,112 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 5eaaa9d25e2cd5c3d12b429e77fa393b/C is initiating minor compaction (all files) 2024-11-13T05:51:14,112 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaaa9d25e2cd5c3d12b429e77fa393b/C in TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,112 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ea8b07d0327f47ecb7c61b5ccc0ce807, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/e17f1d6c0176475da2ac8475bcfff47a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1f1abc49043740bf9713fcd3bba59385] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp, totalSize=36.9 K 2024-11-13T05:51:14,113 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8b07d0327f47ecb7c61b5ccc0ce807, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1731477068495 2024-11-13T05:51:14,113 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e17f1d6c0176475da2ac8475bcfff47a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731477069680 2024-11-13T05:51:14,113 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1abc49043740bf9713fcd3bba59385, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731477070997 2024-11-13T05:51:14,116 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:14,116 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:51:14,116 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 5eaaa9d25e2cd5c3d12b429e77fa393b, disabling compactions & flushes 2024-11-13T05:51:14,116 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,125 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bb43831b37d748c1a858f2d6a215b911_5eaaa9d25e2cd5c3d12b429e77fa393b, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:14,125 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb43831b37d748c1a858f2d6a215b911_5eaaa9d25e2cd5c3d12b429e77fa393b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:14,127 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#C#compaction#200 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:14,128 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3a2c675b44924a26a884a3a16b72eba3 is 50, key is test_row_0/C:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:14,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742068_1244 (size=13255) 2024-11-13T05:51:14,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742067_1243 (size=4469) 2024-11-13T05:51:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:14,495 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:51:14,545 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaaa9d25e2cd5c3d12b429e77fa393b#A#compaction#199 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:14,546 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/3d460fd96d994e26899e5557ee153509 is 175, key is test_row_0/A:col10/1731477071000/Put/seqid=0 2024-11-13T05:51:14,551 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/3a2c675b44924a26a884a3a16b72eba3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a2c675b44924a26a884a3a16b72eba3 2024-11-13T05:51:14,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742069_1245 (size=32209) 2024-11-13T05:51:14,556 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/C of 5eaaa9d25e2cd5c3d12b429e77fa393b into 3a2c675b44924a26a884a3a16b72eba3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:14,556 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:14,556 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/C, priority=13, startTime=1731477074062; duration=0sec 2024-11-13T05:51:14,556 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:14,556 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:C 2024-11-13T05:51:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:14,960 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/3d460fd96d994e26899e5557ee153509 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/3d460fd96d994e26899e5557ee153509 2024-11-13T05:51:14,966 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaaa9d25e2cd5c3d12b429e77fa393b/A of 5eaaa9d25e2cd5c3d12b429e77fa393b into 3d460fd96d994e26899e5557ee153509(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:14,966 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: 2024-11-13T05:51:14,966 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., storeName=5eaaa9d25e2cd5c3d12b429e77fa393b/A, priority=13, startTime=1731477074062; duration=0sec 2024-11-13T05:51:14,966 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,966 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,966 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:14,966 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaaa9d25e2cd5c3d12b429e77fa393b:A 2024-11-13T05:51:14,966 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. after waiting 0 ms 2024-11-13T05:51:14,966 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:14,966 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-13T05:51:14,966 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=A 2024-11-13T05:51:14,966 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:14,966 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=B 2024-11-13T05:51:14,967 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:14,967 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eaaa9d25e2cd5c3d12b429e77fa393b, store=C 2024-11-13T05:51:14,967 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:14,972 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131d3dbf1b26544d659f2091827b045b80_5eaaa9d25e2cd5c3d12b429e77fa393b is 50, key is test_row_1/A:col10/1731477072512/Put/seqid=0 2024-11-13T05:51:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742070_1246 (size=9914) 2024-11-13T05:51:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:15,377 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:15,381 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131d3dbf1b26544d659f2091827b045b80_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131d3dbf1b26544d659f2091827b045b80_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:15,382 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ae16095a241a43cfa15e23ddd4a53291, store: [table=TestAcidGuarantees family=A region=5eaaa9d25e2cd5c3d12b429e77fa393b] 2024-11-13T05:51:15,383 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ae16095a241a43cfa15e23ddd4a53291 is 175, key is test_row_1/A:col10/1731477072512/Put/seqid=0 2024-11-13T05:51:15,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742071_1247 (size=22561) 2024-11-13T05:51:15,789 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=558, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ae16095a241a43cfa15e23ddd4a53291 2024-11-13T05:51:15,802 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/a647caf025e4429388ef752bd37593f3 is 50, key is test_row_1/B:col10/1731477072512/Put/seqid=0 2024-11-13T05:51:15,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742072_1248 (size=9857) 2024-11-13T05:51:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:16,207 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/a647caf025e4429388ef752bd37593f3 2024-11-13T05:51:16,213 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/2b1818d17b564bb88e9fd8e56e4fad45 is 50, key is test_row_1/C:col10/1731477072512/Put/seqid=0 2024-11-13T05:51:16,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742073_1249 (size=9857) 2024-11-13T05:51:16,618 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/2b1818d17b564bb88e9fd8e56e4fad45 2024-11-13T05:51:16,625 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/A/ae16095a241a43cfa15e23ddd4a53291 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ae16095a241a43cfa15e23ddd4a53291 2024-11-13T05:51:16,632 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ae16095a241a43cfa15e23ddd4a53291, entries=100, sequenceid=558, filesize=22.0 K 2024-11-13T05:51:16,633 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/B/a647caf025e4429388ef752bd37593f3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/a647caf025e4429388ef752bd37593f3 2024-11-13T05:51:16,638 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/a647caf025e4429388ef752bd37593f3, entries=100, sequenceid=558, filesize=9.6 K 2024-11-13T05:51:16,639 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/.tmp/C/2b1818d17b564bb88e9fd8e56e4fad45 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/2b1818d17b564bb88e9fd8e56e4fad45 2024-11-13T05:51:16,643 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/2b1818d17b564bb88e9fd8e56e4fad45, entries=100, sequenceid=558, filesize=9.6 K 2024-11-13T05:51:16,644 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1678ms, sequenceid=558, compaction requested=false 2024-11-13T05:51:16,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/8bea3cb1c57c44a6b2eecb15b47e5218, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/18d14ee7d19448f08bcf1f37631cf382, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/6563e3bcef9c4fda82b8d659171953cf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/497179ddeae942fb94a2ae4a8eb3e5d6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/86b03d8f45c14504a9782f7ebd4e24e2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f2f3b15e85da4d4195167fad80ae570d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/a56c8de73e2c47fb85c04d1f9c888489, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee] to archive 2024-11-13T05:51:16,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:51:16,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/af09412b35354aa48e2c94059056fb81 2024-11-13T05:51:16,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/7aa6200fa8334302902158378ba550c4 2024-11-13T05:51:16,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/45808d2342e24953b38e460d869d73d2 2024-11-13T05:51:16,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/08f777ca282641d1b4e6ffb14d809753 2024-11-13T05:51:16,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/8bea3cb1c57c44a6b2eecb15b47e5218 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/8bea3cb1c57c44a6b2eecb15b47e5218 2024-11-13T05:51:16,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/e2381b9c8d534922ac275090078eb193 2024-11-13T05:51:16,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/40ca86318ede437493496eb2188108c4 2024-11-13T05:51:16,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/256343c001ce404cb47be0a7b3d0c0f4 2024-11-13T05:51:16,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/18d14ee7d19448f08bcf1f37631cf382 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/18d14ee7d19448f08bcf1f37631cf382 2024-11-13T05:51:16,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/03bb84288821482085db82cc1d43b722 2024-11-13T05:51:16,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/974a17a4fc884808b2066cd4bcb593a5 2024-11-13T05:51:16,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/61c0954fe3e04633a7371ee7e92f7cb1 2024-11-13T05:51:16,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/6563e3bcef9c4fda82b8d659171953cf to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/6563e3bcef9c4fda82b8d659171953cf 2024-11-13T05:51:16,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/55b46bc1a4ba4a3cb548f0ca1a186bf8 2024-11-13T05:51:16,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/85d1652015f6443ca7db3a5a94b3a6b3 2024-11-13T05:51:16,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/497179ddeae942fb94a2ae4a8eb3e5d6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/497179ddeae942fb94a2ae4a8eb3e5d6 2024-11-13T05:51:16,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/5529de4e345d420698c80ce3a8fb807b 2024-11-13T05:51:16,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/98c401d6590c4c469b02369d8ffd3d47 2024-11-13T05:51:16,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/86b03d8f45c14504a9782f7ebd4e24e2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/86b03d8f45c14504a9782f7ebd4e24e2 2024-11-13T05:51:16,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ea65a164e9614d768d53dfbb8ae62eee 2024-11-13T05:51:16,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/c267906d7ecf4ff784a26137d9b571a7 2024-11-13T05:51:16,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f2f3b15e85da4d4195167fad80ae570d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f2f3b15e85da4d4195167fad80ae570d 2024-11-13T05:51:16,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/0e56b5ee8aa249f5a5b15a856b072d5b 2024-11-13T05:51:16,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/51731b63f3294fa2850ccc85d921cdfa 2024-11-13T05:51:16,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/f15c0bececd04976b88282ec23419132 2024-11-13T05:51:16,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/a56c8de73e2c47fb85c04d1f9c888489 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/a56c8de73e2c47fb85c04d1f9c888489 2024-11-13T05:51:16,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/19444c4f127d486483c080a31b1e73bc 2024-11-13T05:51:16,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/607033f1f2af48a6a276757765769cee 2024-11-13T05:51:16,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/563a52581ccf415d97d0f03c872eb39a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/47588702f9c948078717bad851c29bcf, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/647e901813ac40babed06816291a2c6e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5d5f5986213d4d779dbf3434575d0c08, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/578b2ac2c4a94a64b65fc5fe61a2816f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/b7754855660742cb84a2eb38bb405d1b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3e5cb7f8969f4a79a72d41b221c6ce36, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f94accc162ce4eff9c8ccc1b13c93adb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3f7cd668866e47f8ae5209b8c5346e9f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/7828b7e107eb43bfa27ea19638b809b6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6ed92cca4bca4c6d96fc595f42ff0320, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/8539d6fd74c0420898a1ad7dfb96753e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/0aec00c01c0f4bb48d5a605ca55c9470, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/2a583ee0e2c54f7db15ff8bac9afdc13, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/4a8a16d43f19449d8d176acb72107194, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5b352ef7a87b4bb28c7bc49ac9ce6097, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/08b6fbaf3fe84c6c8583bb080af22232, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/54f6256e44654458b8c1be88a08ede87, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5f635957bd6e4b60b8ce7dfbb510bfab, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/d17dcd50194040608483d45bc11597f9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/ed08a80c8ec84e4398aa05e3ccfc17b7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/40b6089035864d718048379af9457e3a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6f5a33e455e34dbfbc309a1bafa6b69b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e36ddf1cbcb54b4ebd724c4d108c0821, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e1435fae6383454dae5512fbbb1b6ee8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/c07f699c6d0940e687d153b1a5fd2572, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5489d80f6bbb4ac4abb71bdfb0876335, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f0908915f2a0467592ca4adfdcbc2bf2] to archive 2024-11-13T05:51:16,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:51:16,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/563a52581ccf415d97d0f03c872eb39a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/563a52581ccf415d97d0f03c872eb39a 2024-11-13T05:51:16,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/47588702f9c948078717bad851c29bcf to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/47588702f9c948078717bad851c29bcf 2024-11-13T05:51:16,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/647e901813ac40babed06816291a2c6e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/647e901813ac40babed06816291a2c6e 2024-11-13T05:51:16,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5d5f5986213d4d779dbf3434575d0c08 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5d5f5986213d4d779dbf3434575d0c08 2024-11-13T05:51:16,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/578b2ac2c4a94a64b65fc5fe61a2816f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/578b2ac2c4a94a64b65fc5fe61a2816f 2024-11-13T05:51:16,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/b7754855660742cb84a2eb38bb405d1b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/b7754855660742cb84a2eb38bb405d1b 2024-11-13T05:51:16,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3e5cb7f8969f4a79a72d41b221c6ce36 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3e5cb7f8969f4a79a72d41b221c6ce36 2024-11-13T05:51:16,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f94accc162ce4eff9c8ccc1b13c93adb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f94accc162ce4eff9c8ccc1b13c93adb 2024-11-13T05:51:16,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3f7cd668866e47f8ae5209b8c5346e9f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/3f7cd668866e47f8ae5209b8c5346e9f 2024-11-13T05:51:16,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/7828b7e107eb43bfa27ea19638b809b6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/7828b7e107eb43bfa27ea19638b809b6 2024-11-13T05:51:16,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6ed92cca4bca4c6d96fc595f42ff0320 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6ed92cca4bca4c6d96fc595f42ff0320 2024-11-13T05:51:16,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/8539d6fd74c0420898a1ad7dfb96753e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/8539d6fd74c0420898a1ad7dfb96753e 2024-11-13T05:51:16,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/0aec00c01c0f4bb48d5a605ca55c9470 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/0aec00c01c0f4bb48d5a605ca55c9470 2024-11-13T05:51:16,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/2a583ee0e2c54f7db15ff8bac9afdc13 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/2a583ee0e2c54f7db15ff8bac9afdc13 2024-11-13T05:51:16,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/4a8a16d43f19449d8d176acb72107194 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/4a8a16d43f19449d8d176acb72107194 2024-11-13T05:51:16,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5b352ef7a87b4bb28c7bc49ac9ce6097 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5b352ef7a87b4bb28c7bc49ac9ce6097 2024-11-13T05:51:16,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/08b6fbaf3fe84c6c8583bb080af22232 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/08b6fbaf3fe84c6c8583bb080af22232 2024-11-13T05:51:16,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/54f6256e44654458b8c1be88a08ede87 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/54f6256e44654458b8c1be88a08ede87 2024-11-13T05:51:16,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5f635957bd6e4b60b8ce7dfbb510bfab to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5f635957bd6e4b60b8ce7dfbb510bfab 2024-11-13T05:51:16,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/d17dcd50194040608483d45bc11597f9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/d17dcd50194040608483d45bc11597f9 2024-11-13T05:51:16,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/ed08a80c8ec84e4398aa05e3ccfc17b7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/ed08a80c8ec84e4398aa05e3ccfc17b7 2024-11-13T05:51:16,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/40b6089035864d718048379af9457e3a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/40b6089035864d718048379af9457e3a 2024-11-13T05:51:16,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6f5a33e455e34dbfbc309a1bafa6b69b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/6f5a33e455e34dbfbc309a1bafa6b69b 2024-11-13T05:51:16,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e36ddf1cbcb54b4ebd724c4d108c0821 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e36ddf1cbcb54b4ebd724c4d108c0821 2024-11-13T05:51:16,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e1435fae6383454dae5512fbbb1b6ee8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/e1435fae6383454dae5512fbbb1b6ee8 2024-11-13T05:51:16,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/c07f699c6d0940e687d153b1a5fd2572 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/c07f699c6d0940e687d153b1a5fd2572 2024-11-13T05:51:16,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5489d80f6bbb4ac4abb71bdfb0876335 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/5489d80f6bbb4ac4abb71bdfb0876335 2024-11-13T05:51:16,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f0908915f2a0467592ca4adfdcbc2bf2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/f0908915f2a0467592ca4adfdcbc2bf2 2024-11-13T05:51:16,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/71f595a214254acc982d5f3cbd879020, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/a14eb3d7205e486884b6df46136585c5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b690e14b23b7431584e7e74578a7595d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1d4afa228ec5485bad4863d197439f30, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/fb9b71f7c5a1416eb5c5415e18367439, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/27fbe7472f2c4dfd843dd8ba3603d9f8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a4e56b73daa49bdb73b4f4c827b6bab, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3adcc9a742d04ef48a3277e8ef6e2690, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ddeec72a355e47d4af6f31bb0d0c7b30, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1a84b4ff9dba4a3daf414ba99930e63d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b1b0bd810ef54f33a8a8d9c8feab1811, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3bc691a20ebe497497fac5ba633fff56, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/58cf8051aefc49629b115d63e27262e6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab25390d09834ac38b9ec15c7c011c3c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1cbdcf0440ea47bf805b60f71d6857e8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/4fad93b6d79a45baabe99addecc38e06, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/f6982cca69d842e29b0c06e4e8fc32c6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ce2744d3cab74701ae4e404790676939, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/446ce01e29a44be0a04583ed0eeb2b37, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/5595bf7b22454a15b854bb2fbef62f4b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab53980e033942f891a084c7b98bd0ef, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/7c9ce20c2b6a44e79920471fc25026fa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/249f2f989c5b4fe0bb2c491eb5f8aaf0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/8a6e1a3300014ca6939fb48e7fa846c4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ea8b07d0327f47ecb7c61b5ccc0ce807, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/80cc3e393042450b8b7892c6b427a25f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/e17f1d6c0176475da2ac8475bcfff47a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1f1abc49043740bf9713fcd3bba59385] to archive 2024-11-13T05:51:16,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:51:16,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/71f595a214254acc982d5f3cbd879020 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/71f595a214254acc982d5f3cbd879020 2024-11-13T05:51:16,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/a14eb3d7205e486884b6df46136585c5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/a14eb3d7205e486884b6df46136585c5 2024-11-13T05:51:16,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b690e14b23b7431584e7e74578a7595d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b690e14b23b7431584e7e74578a7595d 2024-11-13T05:51:16,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1d4afa228ec5485bad4863d197439f30 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1d4afa228ec5485bad4863d197439f30 2024-11-13T05:51:16,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/fb9b71f7c5a1416eb5c5415e18367439 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/fb9b71f7c5a1416eb5c5415e18367439 2024-11-13T05:51:16,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/27fbe7472f2c4dfd843dd8ba3603d9f8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/27fbe7472f2c4dfd843dd8ba3603d9f8 2024-11-13T05:51:16,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a4e56b73daa49bdb73b4f4c827b6bab to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a4e56b73daa49bdb73b4f4c827b6bab 2024-11-13T05:51:16,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3adcc9a742d04ef48a3277e8ef6e2690 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3adcc9a742d04ef48a3277e8ef6e2690 2024-11-13T05:51:16,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ddeec72a355e47d4af6f31bb0d0c7b30 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ddeec72a355e47d4af6f31bb0d0c7b30 2024-11-13T05:51:16,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1a84b4ff9dba4a3daf414ba99930e63d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1a84b4ff9dba4a3daf414ba99930e63d 2024-11-13T05:51:16,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b1b0bd810ef54f33a8a8d9c8feab1811 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/b1b0bd810ef54f33a8a8d9c8feab1811 2024-11-13T05:51:16,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3bc691a20ebe497497fac5ba633fff56 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3bc691a20ebe497497fac5ba633fff56 2024-11-13T05:51:16,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/58cf8051aefc49629b115d63e27262e6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/58cf8051aefc49629b115d63e27262e6 2024-11-13T05:51:16,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab25390d09834ac38b9ec15c7c011c3c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab25390d09834ac38b9ec15c7c011c3c 2024-11-13T05:51:16,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1cbdcf0440ea47bf805b60f71d6857e8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1cbdcf0440ea47bf805b60f71d6857e8 2024-11-13T05:51:16,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/4fad93b6d79a45baabe99addecc38e06 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/4fad93b6d79a45baabe99addecc38e06 2024-11-13T05:51:16,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/f6982cca69d842e29b0c06e4e8fc32c6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/f6982cca69d842e29b0c06e4e8fc32c6 2024-11-13T05:51:16,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ce2744d3cab74701ae4e404790676939 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ce2744d3cab74701ae4e404790676939 2024-11-13T05:51:16,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/446ce01e29a44be0a04583ed0eeb2b37 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/446ce01e29a44be0a04583ed0eeb2b37 2024-11-13T05:51:16,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/5595bf7b22454a15b854bb2fbef62f4b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/5595bf7b22454a15b854bb2fbef62f4b 2024-11-13T05:51:16,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab53980e033942f891a084c7b98bd0ef to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ab53980e033942f891a084c7b98bd0ef 2024-11-13T05:51:16,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/7c9ce20c2b6a44e79920471fc25026fa to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/7c9ce20c2b6a44e79920471fc25026fa 2024-11-13T05:51:16,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/249f2f989c5b4fe0bb2c491eb5f8aaf0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/249f2f989c5b4fe0bb2c491eb5f8aaf0 2024-11-13T05:51:16,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/8a6e1a3300014ca6939fb48e7fa846c4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/8a6e1a3300014ca6939fb48e7fa846c4 2024-11-13T05:51:16,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ea8b07d0327f47ecb7c61b5ccc0ce807 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/ea8b07d0327f47ecb7c61b5ccc0ce807 2024-11-13T05:51:16,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/80cc3e393042450b8b7892c6b427a25f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/80cc3e393042450b8b7892c6b427a25f 2024-11-13T05:51:16,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/e17f1d6c0176475da2ac8475bcfff47a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/e17f1d6c0176475da2ac8475bcfff47a 2024-11-13T05:51:16,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1f1abc49043740bf9713fcd3bba59385 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/1f1abc49043740bf9713fcd3bba59385 2024-11-13T05:51:16,729 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/recovered.edits/561.seqid, newMaxSeqId=561, maxSeqId=4 2024-11-13T05:51:16,730 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. 2024-11-13T05:51:16,730 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 5eaaa9d25e2cd5c3d12b429e77fa393b: Waiting for close lock at 1731477074116Running coprocessor pre-close hooks at 1731477074116Disabling compacts and flushes for region at 1731477074116Disabling writes for close at 1731477074966 (+850 ms)Obtaining lock to block concurrent updates at 1731477074966Preparing flush snapshotting stores in 5eaaa9d25e2cd5c3d12b429e77fa393b at 1731477074966Finished memstore snapshotting TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731477074967 (+1 ms)Flushing stores of TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b. at 1731477074967Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/A: creating writer at 1731477074967Flushing mob file 5eaaa9d25e2cd5c3d12b429e77fa393b/A: appending metadata at 1731477074972 (+5 ms)Flushing mob file 5eaaa9d25e2cd5c3d12b429e77fa393b/A: closing flushed file at 1731477074972Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/A: appending metadata at 1731477075382 (+410 ms)Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/A: closing flushed file at 1731477075382Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/B: creating writer at 1731477075799 (+417 ms)Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/B: appending metadata at 1731477075802 (+3 ms)Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/B: closing flushed file at 1731477075802Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/C: creating writer at 1731477076210 (+408 ms)Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/C: appending metadata at 1731477076212 (+2 ms)Flushing 5eaaa9d25e2cd5c3d12b429e77fa393b/C: closing flushed file at 1731477076212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9fa14a2: reopening flushed file at 1731477076623 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11136deb: reopening flushed file at 1731477076632 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64bf7740: reopening flushed file at 1731477076638 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5eaaa9d25e2cd5c3d12b429e77fa393b in 1678ms, sequenceid=558, compaction requested=false at 1731477076644 (+6 ms)Writing region close event to WAL at 1731477076726 (+82 ms)Running coprocessor post-close hooks at 1731477076729 (+3 ms)Closed at 1731477076730 (+1 ms) 2024-11-13T05:51:16,732 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:16,732 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5eaaa9d25e2cd5c3d12b429e77fa393b, regionState=CLOSED 2024-11-13T05:51:16,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:51:16,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T05:51:16,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 5eaaa9d25e2cd5c3d12b429e77fa393b, server=bc386b7bf2fe,38979,1731477016232 in 2.7710 sec 2024-11-13T05:51:16,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-13T05:51:16,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaaa9d25e2cd5c3d12b429e77fa393b, UNASSIGN in 2.7760 sec 2024-11-13T05:51:16,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T05:51:16,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7790 sec 2024-11-13T05:51:16,740 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477076740"}]},"ts":"1731477076740"} 2024-11-13T05:51:16,742 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:51:16,742 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:51:16,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7890 sec 2024-11-13T05:51:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T05:51:18,123 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:18,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:51:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,129 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:51:18,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,134 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/recovered.edits] 2024-11-13T05:51:18,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/3d460fd96d994e26899e5557ee153509 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/3d460fd96d994e26899e5557ee153509 2024-11-13T05:51:18,144 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ae16095a241a43cfa15e23ddd4a53291 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/A/ae16095a241a43cfa15e23ddd4a53291 2024-11-13T05:51:18,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/a647caf025e4429388ef752bd37593f3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/a647caf025e4429388ef752bd37593f3 2024-11-13T05:51:18,150 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/bd33fdd48cf74c7492d42d2406c32280 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/B/bd33fdd48cf74c7492d42d2406c32280 2024-11-13T05:51:18,154 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/2b1818d17b564bb88e9fd8e56e4fad45 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/2b1818d17b564bb88e9fd8e56e4fad45 2024-11-13T05:51:18,156 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a2c675b44924a26a884a3a16b72eba3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/C/3a2c675b44924a26a884a3a16b72eba3 2024-11-13T05:51:18,160 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/recovered.edits/561.seqid to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b/recovered.edits/561.seqid 2024-11-13T05:51:18,161 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,161 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:51:18,161 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:51:18,162 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T05:51:18,166 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111306b2b6d85e004d1db6360da9d2a41ed2_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111306b2b6d85e004d1db6360da9d2a41ed2_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,167 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113143711942afa4cf6a27097d8ffbd7f5f_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113143711942afa4cf6a27097d8ffbd7f5f_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,168 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131d3dbf1b26544d659f2091827b045b80_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131d3dbf1b26544d659f2091827b045b80_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,169 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113249394841a744ba38fc8cda51ef13740_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113249394841a744ba38fc8cda51ef13740_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,170 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132e24f623bc6941ed8b26c9c2ec1a97a6_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132e24f623bc6941ed8b26c9c2ec1a97a6_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,171 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f237e6579804121933b71f6bfed0595_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f237e6579804121933b71f6bfed0595_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,172 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111336b4a16cfdd14d06b55a4d0e448c4cee_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111336b4a16cfdd14d06b55a4d0e448c4cee_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,173 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133fc079fa49fd444c81bce7075ef4a513_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133fc079fa49fd444c81bce7075ef4a513_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,174 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343ec765f145f4aeeb7c9a85dcec9b20d_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343ec765f145f4aeeb7c9a85dcec9b20d_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,175 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113510223fc5ce44e639f0d35f7621328df_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113510223fc5ce44e639f0d35f7621328df_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,176 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113531b404a2498491b88698c5083eeb946_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113531b404a2498491b88698c5083eeb946_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,177 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113563b4e9f591e4c5198012e68fe8ed439_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113563b4e9f591e4c5198012e68fe8ed439_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136db3f7af0482485c9ebc92bd889619a8_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136db3f7af0482485c9ebc92bd889619a8_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,179 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b7af7a501664b0a881df1f6d6f466a8_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137b7af7a501664b0a881df1f6d6f466a8_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,180 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138159ff99ce7c4e62be9da341363a6771_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138159ff99ce7c4e62be9da341363a6771_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,181 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111382fdaafdd2a34ff6af6c222a743a33ad_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111382fdaafdd2a34ff6af6c222a743a33ad_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,182 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391d05a0610d74e27a245b63210f69cab_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111391d05a0610d74e27a245b63210f69cab_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,183 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a077b8b1a4b9447c99ea94b923dcfaeb_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a077b8b1a4b9447c99ea94b923dcfaeb_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,183 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113adc856147b0444ce9e0446a3ee8cfef5_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113adc856147b0444ce9e0446a3ee8cfef5_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,184 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b77b95f84f184833822e43e20a021932_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b77b95f84f184833822e43e20a021932_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,185 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de6658d6f0e44e528d1cbead502d104d_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113de6658d6f0e44e528d1cbead502d104d_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,186 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e3eca15c2cf94aaa9f4504165105f54e_5eaaa9d25e2cd5c3d12b429e77fa393b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e3eca15c2cf94aaa9f4504165105f54e_5eaaa9d25e2cd5c3d12b429e77fa393b 2024-11-13T05:51:18,187 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:51:18,188 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,190 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:51:18,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:51:18,193 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,193 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:51:18,194 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731477078193"}]},"ts":"9223372036854775807"} 2024-11-13T05:51:18,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:51:18,195 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5eaaa9d25e2cd5c3d12b429e77fa393b, NAME => 'TestAcidGuarantees,,1731477048087.5eaaa9d25e2cd5c3d12b429e77fa393b.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:51:18,195 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:51:18,196 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731477078195"}]},"ts":"9223372036854775807"} 2024-11-13T05:51:18,197 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:51:18,198 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-13T05:51:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T05:51:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:51:18,242 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:18,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 248) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-820210548_22 at /127.0.0.1:51554 [Waiting for operation #653] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1749926084_22 at /127.0.0.1:51550 [Waiting for operation #686] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-820210548_22 at /127.0.0.1:33216 [Waiting for operation #1099] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1749926084_22 at /127.0.0.1:58596 [Waiting for operation #836] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 459) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7190 (was 7132) - AvailableMemoryMB LEAK? - 2024-11-13T05:51:18,259 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=475, ProcessCount=11, AvailableMemoryMB=7190 2024-11-13T05:51:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:51:18,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:51:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:18,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:51:18,263 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:18,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-13T05:51:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T05:51:18,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:51:18,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742074_1250 (size=1061) 2024-11-13T05:51:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T05:51:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T05:51:18,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:51:18,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742075_1251 (size=53) 2024-11-13T05:51:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T05:51:19,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:51:19,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f3802ebb1be649f045361204f3bf7507, disabling compactions & flushes 2024-11-13T05:51:19,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. after waiting 0 ms 2024-11-13T05:51:19,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f3802ebb1be649f045361204f3bf7507: Waiting for close lock at 1731477079090Disabling compacts and flushes for region at 1731477079090Disabling writes for close at 1731477079090Writing region close event to WAL at 1731477079090Closed at 1731477079090 2024-11-13T05:51:19,092 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:51:19,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731477079092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477079092"}]},"ts":"1731477079092"} 2024-11-13T05:51:19,096 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:51:19,097 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:51:19,098 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477079097"}]},"ts":"1731477079097"} 2024-11-13T05:51:19,100 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:51:19,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, ASSIGN}] 2024-11-13T05:51:19,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, ASSIGN 2024-11-13T05:51:19,104 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=false 2024-11-13T05:51:19,255 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f3802ebb1be649f045361204f3bf7507, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:19,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, ASSIGN because future has completed 2024-11-13T05:51:19,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:51:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T05:51:19,417 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,417 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:51:19,418 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,418 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:51:19,418 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,419 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,422 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,423 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:19,423 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3802ebb1be649f045361204f3bf7507 columnFamilyName A 2024-11-13T05:51:19,423 DEBUG [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:19,424 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.HStore(327): Store=f3802ebb1be649f045361204f3bf7507/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:19,424 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,425 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:19,425 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3802ebb1be649f045361204f3bf7507 columnFamilyName B 2024-11-13T05:51:19,426 DEBUG [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:19,426 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.HStore(327): Store=f3802ebb1be649f045361204f3bf7507/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:19,426 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,427 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:19,427 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3802ebb1be649f045361204f3bf7507 columnFamilyName C 2024-11-13T05:51:19,428 DEBUG [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:19,428 INFO [StoreOpener-f3802ebb1be649f045361204f3bf7507-1 {}] regionserver.HStore(327): Store=f3802ebb1be649f045361204f3bf7507/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:19,428 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,428 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,429 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,429 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,430 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,431 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,431 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:51:19,433 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,435 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:51:19,435 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened f3802ebb1be649f045361204f3bf7507; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73473933, jitterRate=0.09484691917896271}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:51:19,436 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:19,438 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for f3802ebb1be649f045361204f3bf7507: Running coprocessor pre-open hook at 1731477079419Writing region info on filesystem at 1731477079419Initializing all the Stores at 1731477079421 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477079421Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477079422 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477079422Cleaning up temporary data from old regions at 1731477079431 (+9 ms)Running coprocessor post-open hooks at 1731477079436 (+5 ms)Region opened successfully at 1731477079437 (+1 ms) 2024-11-13T05:51:19,438 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., pid=67, masterSystemTime=1731477079410 2024-11-13T05:51:19,441 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,441 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:19,441 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f3802ebb1be649f045361204f3bf7507, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:19,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:51:19,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T05:51:19,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 in 187 msec 2024-11-13T05:51:19,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-13T05:51:19,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, ASSIGN in 345 msec 2024-11-13T05:51:19,450 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:51:19,450 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477079450"}]},"ts":"1731477079450"} 2024-11-13T05:51:19,452 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:51:19,453 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:51:19,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-13T05:51:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-13T05:51:20,413 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:20,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429799b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d02721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,422 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f03c2de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:51:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:51:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@900e1fd 2024-11-13T05:51:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:51:20,429 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:51:20,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6327b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,430 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,431 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c44e9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,433 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eba13c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fdf952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,435 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8219ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b391da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b624df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,437 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d99dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a2e9ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f6d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,440 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbcaea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9a5579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00a41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,442 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3de994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,444 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138396d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bafd9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,445 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b3ba02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3fa93b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63eadf7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,448 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152c436f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,450 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50012a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28097ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,452 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa03217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f68f839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dda73f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c956b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395cdc62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48dcb24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,458 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:20,458 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:20,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55064a92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:20,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336a951a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:20,461 DEBUG [Thread-1173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,461 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,461 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,461 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,461 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,461 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,461 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,462 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,462 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,464 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:20,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,465 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:20,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T05:51:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:51:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,470 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,472 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:20,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:20,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:51:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:20,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:51:20,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:20,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:20,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:20,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:20,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:20,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:20,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2b97ee8e783c4ad2975ab1c152eb3e57 is 50, key is test_row_0/A:col10/1731477080474/Put/seqid=0 2024-11-13T05:51:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477140503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477080497, max timestamp=1731477080508Num uniques 859; 2024-11-13T05:51:20,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477140507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477080497, max timestamp=1731477080508Num uniques 859; 2024-11-13T05:51:20,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477080497, max timestamp=1731477080508Num uniques 859; 2024-11-13T05:51:20,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477140538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477140539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477140540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742076_1252 (size=12001) 2024-11-13T05:51:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:51:20,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477140621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477140621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:51:20,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:20,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:20,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:20,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477140652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477140652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477140652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:51:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:20,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:51:20,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477140832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477140832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477140861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477140861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477140861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:20,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:51:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:20,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:20,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2b97ee8e783c4ad2975ab1c152eb3e57 2024-11-13T05:51:20,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/ea28fda89b494421b532b62dc821286d is 50, key is test_row_0/B:col10/1731477080474/Put/seqid=0 2024-11-13T05:51:21,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742077_1253 (size=12001) 2024-11-13T05:51:21,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/ea28fda89b494421b532b62dc821286d 2024-11-13T05:51:21,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/f8c3637cbc7f4b8d9e51862d37b86c90 is 50, key is test_row_0/C:col10/1731477080474/Put/seqid=0 2024-11-13T05:51:21,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742078_1254 (size=12001) 2024-11-13T05:51:21,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/f8c3637cbc7f4b8d9e51862d37b86c90 2024-11-13T05:51:21,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2b97ee8e783c4ad2975ab1c152eb3e57 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b97ee8e783c4ad2975ab1c152eb3e57 2024-11-13T05:51:21,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b97ee8e783c4ad2975ab1c152eb3e57, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T05:51:21,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/ea28fda89b494421b532b62dc821286d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ea28fda89b494421b532b62dc821286d 2024-11-13T05:51:21,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ea28fda89b494421b532b62dc821286d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T05:51:21,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/f8c3637cbc7f4b8d9e51862d37b86c90 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f8c3637cbc7f4b8d9e51862d37b86c90 2024-11-13T05:51:21,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f8c3637cbc7f4b8d9e51862d37b86c90, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T05:51:21,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f3802ebb1be649f045361204f3bf7507 in 608ms, sequenceid=12, compaction requested=false 2024-11-13T05:51:21,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:21,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T05:51:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:21,085 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T05:51:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:21,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/a101445c95084b78997f964f779721da is 50, key is test_row_0/A:col10/1731477080538/Put/seqid=0 2024-11-13T05:51:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:51:21,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742079_1255 (size=12001) 2024-11-13T05:51:21,098 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/a101445c95084b78997f964f779721da 2024-11-13T05:51:21,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/3c517c0b7e194b36924df88ad1497118 is 50, key is test_row_0/B:col10/1731477080538/Put/seqid=0 2024-11-13T05:51:21,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742080_1256 (size=12001) 2024-11-13T05:51:21,134 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/3c517c0b7e194b36924df88ad1497118 2024-11-13T05:51:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:21,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/4404b1b8f70b4e8c8e68d100abb55868 is 50, key is test_row_0/C:col10/1731477080538/Put/seqid=0 2024-11-13T05:51:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742081_1257 (size=12001) 2024-11-13T05:51:21,157 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/4404b1b8f70b4e8c8e68d100abb55868 2024-11-13T05:51:21,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/a101445c95084b78997f964f779721da as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a101445c95084b78997f964f779721da 2024-11-13T05:51:21,173 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a101445c95084b78997f964f779721da, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T05:51:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/3c517c0b7e194b36924df88ad1497118 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3c517c0b7e194b36924df88ad1497118 2024-11-13T05:51:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477141172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477141172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477141172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477141172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477141176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,181 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3c517c0b7e194b36924df88ad1497118, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T05:51:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/4404b1b8f70b4e8c8e68d100abb55868 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4404b1b8f70b4e8c8e68d100abb55868 2024-11-13T05:51:21,189 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4404b1b8f70b4e8c8e68d100abb55868, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T05:51:21,191 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for f3802ebb1be649f045361204f3bf7507 in 106ms, sequenceid=47, compaction requested=false 2024-11-13T05:51:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T05:51:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T05:51:21,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T05:51:21,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 720 msec 2024-11-13T05:51:21,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 732 msec 2024-11-13T05:51:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:21,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T05:51:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:21,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2b88d699eeee4973bb8e30f0cc6c46a3 is 50, key is test_row_0/A:col10/1731477081151/Put/seqid=0 2024-11-13T05:51:21,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742082_1258 (size=12001) 2024-11-13T05:51:21,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477081308, max timestamp=1731477081325Num uniques 859; 2024-11-13T05:51:21,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477081308, max timestamp=1731477081325Num uniques 859; 2024-11-13T05:51:21,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:21,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477081308, max timestamp=1731477081325Num uniques 859; 2024-11-13T05:51:21,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:21,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477141369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477141369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477141481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477141481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T05:51:21,601 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:21,601 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T05:51:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:21,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:21,604 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:21,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477141682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477141682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477141682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477141691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477141691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:21,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2b88d699eeee4973bb8e30f0cc6c46a3 2024-11-13T05:51:21,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/b37ea0871c5c44eca3c7e2c81cb69fad is 50, key is test_row_0/B:col10/1731477081151/Put/seqid=0 2024-11-13T05:51:21,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742083_1259 (size=12001) 2024-11-13T05:51:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:21,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:21,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:21,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:21,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477142002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477142002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/b37ea0871c5c44eca3c7e2c81cb69fad 2024-11-13T05:51:22,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/ab3e5f31d8ef4e7092fc688b8ea0b315 is 50, key is test_row_0/C:col10/1731477081151/Put/seqid=0 2024-11-13T05:51:22,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742084_1260 (size=12001) 2024-11-13T05:51:22,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:22,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:51:22,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477142511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477142511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/ab3e5f31d8ef4e7092fc688b8ea0b315 2024-11-13T05:51:22,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2b88d699eeee4973bb8e30f0cc6c46a3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b88d699eeee4973bb8e30f0cc6c46a3 2024-11-13T05:51:22,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b88d699eeee4973bb8e30f0cc6c46a3, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T05:51:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/b37ea0871c5c44eca3c7e2c81cb69fad as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b37ea0871c5c44eca3c7e2c81cb69fad 2024-11-13T05:51:22,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:22,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:22,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b37ea0871c5c44eca3c7e2c81cb69fad, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T05:51:22,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/ab3e5f31d8ef4e7092fc688b8ea0b315 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/ab3e5f31d8ef4e7092fc688b8ea0b315 2024-11-13T05:51:22,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/ab3e5f31d8ef4e7092fc688b8ea0b315, entries=150, sequenceid=59, filesize=11.7 K 2024-11-13T05:51:22,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for f3802ebb1be649f045361204f3bf7507 in 1260ms, sequenceid=59, compaction requested=true 2024-11-13T05:51:22,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:22,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:22,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:22,543 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:22,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:22,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:22,544 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:22,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:22,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:22,545 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:22,545 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:22,545 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:22,545 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,546 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b97ee8e783c4ad2975ab1c152eb3e57, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a101445c95084b78997f964f779721da, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b88d699eeee4973bb8e30f0cc6c46a3] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=35.2 K 2024-11-13T05:51:22,546 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:22,546 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,546 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ea28fda89b494421b532b62dc821286d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3c517c0b7e194b36924df88ad1497118, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b37ea0871c5c44eca3c7e2c81cb69fad] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=35.2 K 2024-11-13T05:51:22,548 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ea28fda89b494421b532b62dc821286d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731477080474 2024-11-13T05:51:22,548 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b97ee8e783c4ad2975ab1c152eb3e57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731477080474 2024-11-13T05:51:22,549 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c517c0b7e194b36924df88ad1497118, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477080531 2024-11-13T05:51:22,549 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a101445c95084b78997f964f779721da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477080531 2024-11-13T05:51:22,549 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b37ea0871c5c44eca3c7e2c81cb69fad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477081151 2024-11-13T05:51:22,550 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b88d699eeee4973bb8e30f0cc6c46a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477081151 2024-11-13T05:51:22,573 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:22,573 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/b218f581e5774adc97d6ff9342a40ec4 is 50, key is test_row_0/A:col10/1731477081151/Put/seqid=0 2024-11-13T05:51:22,581 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:22,582 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/3203fe8e181c443fa2ff9e805f5b5db0 is 50, key is test_row_0/B:col10/1731477081151/Put/seqid=0 2024-11-13T05:51:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742085_1261 (size=12104) 2024-11-13T05:51:22,606 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/3203fe8e181c443fa2ff9e805f5b5db0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3203fe8e181c443fa2ff9e805f5b5db0 2024-11-13T05:51:22,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742086_1262 (size=12104) 2024-11-13T05:51:22,618 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into 3203fe8e181c443fa2ff9e805f5b5db0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:22,618 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:22,618 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=13, startTime=1731477082543; duration=0sec 2024-11-13T05:51:22,619 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:22,619 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:22,619 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:22,620 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:22,620 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:22,621 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,621 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f8c3637cbc7f4b8d9e51862d37b86c90, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4404b1b8f70b4e8c8e68d100abb55868, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/ab3e5f31d8ef4e7092fc688b8ea0b315] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=35.2 K 2024-11-13T05:51:22,621 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c3637cbc7f4b8d9e51862d37b86c90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731477080474 2024-11-13T05:51:22,621 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4404b1b8f70b4e8c8e68d100abb55868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477080531 2024-11-13T05:51:22,622 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ab3e5f31d8ef4e7092fc688b8ea0b315, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477081151 2024-11-13T05:51:22,628 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:22,629 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/de9caee55791481d8d5e8f97227a355a is 50, key is test_row_0/C:col10/1731477081151/Put/seqid=0 2024-11-13T05:51:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742087_1263 (size=12104) 2024-11-13T05:51:22,648 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/de9caee55791481d8d5e8f97227a355a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/de9caee55791481d8d5e8f97227a355a 2024-11-13T05:51:22,654 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into de9caee55791481d8d5e8f97227a355a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:22,654 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:22,654 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=13, startTime=1731477082544; duration=0sec 2024-11-13T05:51:22,654 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:22,654 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:22,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T05:51:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:22,686 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-13T05:51:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/e18c055660404e17abea3f971c85459c is 50, key is test_row_0/A:col10/1731477081356/Put/seqid=0 2024-11-13T05:51:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:22,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742088_1264 (size=12001) 2024-11-13T05:51:22,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477142708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477142713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477142713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477142821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477142831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477142832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,019 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/b218f581e5774adc97d6ff9342a40ec4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b218f581e5774adc97d6ff9342a40ec4 2024-11-13T05:51:23,023 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into b218f581e5774adc97d6ff9342a40ec4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:23,023 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:23,023 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=13, startTime=1731477082543; duration=0sec 2024-11-13T05:51:23,023 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:23,023 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:23,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477143031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477143041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477143041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,100 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/e18c055660404e17abea3f971c85459c 2024-11-13T05:51:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/7129fb2297414d8da054240d2a4647ed is 50, key is test_row_0/B:col10/1731477081356/Put/seqid=0 2024-11-13T05:51:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742089_1265 (size=12001) 2024-11-13T05:51:23,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477143342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477143351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477143351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,511 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/7129fb2297414d8da054240d2a4647ed 2024-11-13T05:51:23,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/90e14dfabcf54febaa1085ef6081e326 is 50, key is test_row_0/C:col10/1731477081356/Put/seqid=0 2024-11-13T05:51:23,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742090_1266 (size=12001) 2024-11-13T05:51:23,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477143521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,523 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477143531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,535 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:23,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477143851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477143861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477143861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:23,921 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/90e14dfabcf54febaa1085ef6081e326 2024-11-13T05:51:23,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/e18c055660404e17abea3f971c85459c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/e18c055660404e17abea3f971c85459c 2024-11-13T05:51:23,928 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/e18c055660404e17abea3f971c85459c, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T05:51:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/7129fb2297414d8da054240d2a4647ed as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7129fb2297414d8da054240d2a4647ed 2024-11-13T05:51:23,932 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7129fb2297414d8da054240d2a4647ed, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T05:51:23,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/90e14dfabcf54febaa1085ef6081e326 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/90e14dfabcf54febaa1085ef6081e326 2024-11-13T05:51:23,937 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/90e14dfabcf54febaa1085ef6081e326, entries=150, sequenceid=95, filesize=11.7 K 2024-11-13T05:51:23,938 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for f3802ebb1be649f045361204f3bf7507 in 1251ms, sequenceid=95, compaction requested=false 2024-11-13T05:51:23,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:23,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:23,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T05:51:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T05:51:23,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T05:51:23,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3350 sec 2024-11-13T05:51:23,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.3400 sec 2024-11-13T05:51:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:24,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:51:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:24,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c46a4a94a25f41f4bd7ab987d5b911d6 is 50, key is test_row_0/A:col10/1731477084872/Put/seqid=0 2024-11-13T05:51:24,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742091_1267 (size=12001) 2024-11-13T05:51:24,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477144897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477144901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477144901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:24,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477145011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477145012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477145012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477145221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477145221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477145221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c46a4a94a25f41f4bd7ab987d5b911d6 2024-11-13T05:51:25,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/966fb601cbe949809819e5964ff73fb9 is 50, key is test_row_0/B:col10/1731477084872/Put/seqid=0 2024-11-13T05:51:25,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742092_1268 (size=12001) 2024-11-13T05:51:25,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/966fb601cbe949809819e5964ff73fb9 2024-11-13T05:51:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477145531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477145531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477145531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,534 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:25,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477145532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e is 50, key is test_row_0/C:col10/1731477084872/Put/seqid=0 2024-11-13T05:51:25,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742093_1269 (size=12001) 2024-11-13T05:51:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477145552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:25,553 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T05:51:25,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:25,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T05:51:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:25,763 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:25,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:25,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:25,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:25,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:25,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:25,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:25,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:25,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:25,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e 2024-11-13T05:51:25,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c46a4a94a25f41f4bd7ab987d5b911d6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c46a4a94a25f41f4bd7ab987d5b911d6 2024-11-13T05:51:25,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c46a4a94a25f41f4bd7ab987d5b911d6, entries=150, sequenceid=110, filesize=11.7 K 2024-11-13T05:51:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/966fb601cbe949809819e5964ff73fb9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/966fb601cbe949809819e5964ff73fb9 2024-11-13T05:51:25,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/966fb601cbe949809819e5964ff73fb9, entries=150, sequenceid=110, filesize=11.7 K 2024-11-13T05:51:25,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e 2024-11-13T05:51:25,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e, entries=150, sequenceid=110, filesize=11.7 K 2024-11-13T05:51:25,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f3802ebb1be649f045361204f3bf7507 in 1089ms, sequenceid=110, compaction requested=true 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:25,962 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:25,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:25,962 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:25,963 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:25,963 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:25,963 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:25,963 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:25,963 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:25,963 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3203fe8e181c443fa2ff9e805f5b5db0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7129fb2297414d8da054240d2a4647ed, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/966fb601cbe949809819e5964ff73fb9] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=35.3 K 2024-11-13T05:51:25,963 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:25,963 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b218f581e5774adc97d6ff9342a40ec4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/e18c055660404e17abea3f971c85459c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c46a4a94a25f41f4bd7ab987d5b911d6] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=35.3 K 2024-11-13T05:51:25,964 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3203fe8e181c443fa2ff9e805f5b5db0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477081151 2024-11-13T05:51:25,964 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b218f581e5774adc97d6ff9342a40ec4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477081151 2024-11-13T05:51:25,964 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7129fb2297414d8da054240d2a4647ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731477081349 2024-11-13T05:51:25,964 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e18c055660404e17abea3f971c85459c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731477081349 2024-11-13T05:51:25,964 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 966fb601cbe949809819e5964ff73fb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731477082712 2024-11-13T05:51:25,965 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46a4a94a25f41f4bd7ab987d5b911d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731477082712 2024-11-13T05:51:25,981 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:25,981 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:25,981 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/7fd2d43d44bf4d419aae85a3464a93d8 is 50, key is test_row_0/B:col10/1731477084872/Put/seqid=0 2024-11-13T05:51:25,981 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/ee6d05f9e11c4179b1bff600f69a7cd4 is 50, key is test_row_0/A:col10/1731477084872/Put/seqid=0 2024-11-13T05:51:26,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742095_1271 (size=12207) 2024-11-13T05:51:26,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742094_1270 (size=12207) 2024-11-13T05:51:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:26,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-13T05:51:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:26,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:26,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/5357b7555dfb4258a1c59ff63e57eab3 is 50, key is test_row_0/A:col10/1731477084883/Put/seqid=0 2024-11-13T05:51:26,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742096_1272 (size=16781) 2024-11-13T05:51:26,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477146059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477146060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477146067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:26,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477146171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477146171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477146181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477146381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477146382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:26,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477146391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,404 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/ee6d05f9e11c4179b1bff600f69a7cd4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/ee6d05f9e11c4179b1bff600f69a7cd4 2024-11-13T05:51:26,412 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into ee6d05f9e11c4179b1bff600f69a7cd4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:26,412 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:26,412 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=13, startTime=1731477085962; duration=0sec 2024-11-13T05:51:26,412 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:26,412 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:26,412 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:26,413 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/7fd2d43d44bf4d419aae85a3464a93d8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7fd2d43d44bf4d419aae85a3464a93d8 2024-11-13T05:51:26,413 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:26,413 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:26,413 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,413 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/de9caee55791481d8d5e8f97227a355a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/90e14dfabcf54febaa1085ef6081e326, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=35.3 K 2024-11-13T05:51:26,414 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9caee55791481d8d5e8f97227a355a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731477081151 2024-11-13T05:51:26,414 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e14dfabcf54febaa1085ef6081e326, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731477081349 2024-11-13T05:51:26,414 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bccaa3a0dcf4629abaac0d1c8f0fb7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731477082712 2024-11-13T05:51:26,417 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into 7fd2d43d44bf4d419aae85a3464a93d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:26,418 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:26,418 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=13, startTime=1731477085962; duration=0sec 2024-11-13T05:51:26,418 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:26,418 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:26,423 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:26,424 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/115520e507de48d5a63c544d645c270b is 50, key is test_row_0/C:col10/1731477084872/Put/seqid=0 2024-11-13T05:51:26,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742097_1273 (size=12207) 2024-11-13T05:51:26,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/5357b7555dfb4258a1c59ff63e57eab3 2024-11-13T05:51:26,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/f2f634c177d142b1963fe9f4ee3a8a70 is 50, key is test_row_0/B:col10/1731477084883/Put/seqid=0 2024-11-13T05:51:26,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742098_1274 (size=12101) 2024-11-13T05:51:26,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477146692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477146692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477146701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:26,833 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/115520e507de48d5a63c544d645c270b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/115520e507de48d5a63c544d645c270b 2024-11-13T05:51:26,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,841 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into 115520e507de48d5a63c544d645c270b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:26,841 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:26,841 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=13, startTime=1731477085962; duration=0sec 2024-11-13T05:51:26,841 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:26,841 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:26,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/f2f634c177d142b1963fe9f4ee3a8a70 2024-11-13T05:51:26,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b4b78d436c3e480fb21e889b57088179 is 50, key is test_row_0/C:col10/1731477084883/Put/seqid=0 2024-11-13T05:51:26,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742099_1275 (size=12101) 2024-11-13T05:51:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:26,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:26,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:27,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:27,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477147201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:27,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477147201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:27,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477147211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:27,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b4b78d436c3e480fb21e889b57088179 2024-11-13T05:51:27,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/5357b7555dfb4258a1c59ff63e57eab3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/5357b7555dfb4258a1c59ff63e57eab3 2024-11-13T05:51:27,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/5357b7555dfb4258a1c59ff63e57eab3, entries=250, sequenceid=131, filesize=16.4 K 2024-11-13T05:51:27,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/f2f634c177d142b1963fe9f4ee3a8a70 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f2f634c177d142b1963fe9f4ee3a8a70 2024-11-13T05:51:27,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:27,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f2f634c177d142b1963fe9f4ee3a8a70, entries=150, sequenceid=131, filesize=11.8 K 2024-11-13T05:51:27,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:27,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b4b78d436c3e480fb21e889b57088179 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4b78d436c3e480fb21e889b57088179 2024-11-13T05:51:27,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4b78d436c3e480fb21e889b57088179, entries=150, sequenceid=131, filesize=11.8 K 2024-11-13T05:51:27,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f3802ebb1be649f045361204f3bf7507 in 1254ms, sequenceid=131, compaction requested=false 2024-11-13T05:51:27,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:27,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:27,446 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:27,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:27,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/001ea012a05d493385c6cf116c239a4a is 50, key is test_row_0/A:col10/1731477086066/Put/seqid=0 2024-11-13T05:51:27,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742100_1276 (size=12151) 2024-11-13T05:51:27,854 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/001ea012a05d493385c6cf116c239a4a 2024-11-13T05:51:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/bc83be3e014242ac93f0ab08dac6e060 is 50, key is test_row_0/B:col10/1731477086066/Put/seqid=0 2024-11-13T05:51:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742101_1277 (size=12151) 2024-11-13T05:51:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:28,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:28,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477088238, max timestamp=1731477088249Num uniques 859; 2024-11-13T05:51:28,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477088238, max timestamp=1731477088249Num uniques 859; 2024-11-13T05:51:28,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:28,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:28,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731477088238, max timestamp=1731477088249Num uniques 859; 2024-11-13T05:51:28,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-13T05:51:28,265 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/bc83be3e014242ac93f0ab08dac6e060 2024-11-13T05:51:28,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/715fb1f81c004d94982b4839a41b6bb2 is 50, key is test_row_0/C:col10/1731477086066/Put/seqid=0 2024-11-13T05:51:28,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742102_1278 (size=12151) 2024-11-13T05:51:28,277 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/715fb1f81c004d94982b4839a41b6bb2 2024-11-13T05:51:28,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477148276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/001ea012a05d493385c6cf116c239a4a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/001ea012a05d493385c6cf116c239a4a 2024-11-13T05:51:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477148277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477148278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,286 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/001ea012a05d493385c6cf116c239a4a, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T05:51:28,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/bc83be3e014242ac93f0ab08dac6e060 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/bc83be3e014242ac93f0ab08dac6e060 2024-11-13T05:51:28,291 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/bc83be3e014242ac93f0ab08dac6e060, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T05:51:28,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/715fb1f81c004d94982b4839a41b6bb2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/715fb1f81c004d94982b4839a41b6bb2 2024-11-13T05:51:28,296 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/715fb1f81c004d94982b4839a41b6bb2, entries=150, sequenceid=145, filesize=11.9 K 2024-11-13T05:51:28,297 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f3802ebb1be649f045361204f3bf7507 in 851ms, sequenceid=145, compaction requested=true 2024-11-13T05:51:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T05:51:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T05:51:28,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T05:51:28,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-11-13T05:51:28,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5410 sec 2024-11-13T05:51:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:28,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-13T05:51:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:28,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2ea5e959317946b4aff6e2e78da1fd02 is 50, key is test_row_0/A:col10/1731477088277/Put/seqid=0 2024-11-13T05:51:28,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742103_1279 (size=12151) 2024-11-13T05:51:28,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2ea5e959317946b4aff6e2e78da1fd02 2024-11-13T05:51:28,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/abb73ca8b9524211afcba33c092728fd is 50, key is test_row_0/B:col10/1731477088277/Put/seqid=0 2024-11-13T05:51:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477148415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477148415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742104_1280 (size=12151) 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477148415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477148532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477148532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477148532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477148742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477148742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477148742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:28,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/abb73ca8b9524211afcba33c092728fd 2024-11-13T05:51:28,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/6efb87d8b60f4088b7818007f4a7f6a0 is 50, key is test_row_0/C:col10/1731477088277/Put/seqid=0 2024-11-13T05:51:28,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742105_1281 (size=12151) 2024-11-13T05:51:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477149051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477149051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477149051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/6efb87d8b60f4088b7818007f4a7f6a0 2024-11-13T05:51:29,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/2ea5e959317946b4aff6e2e78da1fd02 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2ea5e959317946b4aff6e2e78da1fd02 2024-11-13T05:51:29,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2ea5e959317946b4aff6e2e78da1fd02, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T05:51:29,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/abb73ca8b9524211afcba33c092728fd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/abb73ca8b9524211afcba33c092728fd 2024-11-13T05:51:29,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/abb73ca8b9524211afcba33c092728fd, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T05:51:29,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/6efb87d8b60f4088b7818007f4a7f6a0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/6efb87d8b60f4088b7818007f4a7f6a0 2024-11-13T05:51:29,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/6efb87d8b60f4088b7818007f4a7f6a0, entries=150, sequenceid=180, filesize=11.9 K 2024-11-13T05:51:29,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=60.38 KB/61830 for f3802ebb1be649f045361204f3bf7507 in 855ms, sequenceid=180, compaction requested=true 2024-11-13T05:51:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:29,250 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:29,251 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:29,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:29,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:29,252 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:29,252 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:29,252 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:29,252 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/ee6d05f9e11c4179b1bff600f69a7cd4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/5357b7555dfb4258a1c59ff63e57eab3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/001ea012a05d493385c6cf116c239a4a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2ea5e959317946b4aff6e2e78da1fd02] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=52.0 K 2024-11-13T05:51:29,252 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:29,252 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:29,252 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:29,252 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7fd2d43d44bf4d419aae85a3464a93d8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f2f634c177d142b1963fe9f4ee3a8a70, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/bc83be3e014242ac93f0ab08dac6e060, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/abb73ca8b9524211afcba33c092728fd] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=47.5 K 2024-11-13T05:51:29,253 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6d05f9e11c4179b1bff600f69a7cd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731477082712 2024-11-13T05:51:29,253 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd2d43d44bf4d419aae85a3464a93d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731477082712 2024-11-13T05:51:29,253 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5357b7555dfb4258a1c59ff63e57eab3, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731477084883 2024-11-13T05:51:29,253 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f634c177d142b1963fe9f4ee3a8a70, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731477084883 2024-11-13T05:51:29,254 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001ea012a05d493385c6cf116c239a4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731477086057 2024-11-13T05:51:29,254 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting bc83be3e014242ac93f0ab08dac6e060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731477086057 2024-11-13T05:51:29,254 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea5e959317946b4aff6e2e78da1fd02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477088269 2024-11-13T05:51:29,254 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting abb73ca8b9524211afcba33c092728fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477088269 2024-11-13T05:51:29,269 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:29,270 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/0eb2dbe49f814cb8be4224971a4505b5 is 50, key is test_row_0/A:col10/1731477088277/Put/seqid=0 2024-11-13T05:51:29,286 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#235 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:29,286 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/24ac1f56cda8444098b95877d6f8015c is 50, key is test_row_0/B:col10/1731477088277/Put/seqid=0 2024-11-13T05:51:29,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742106_1282 (size=12493) 2024-11-13T05:51:29,305 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/0eb2dbe49f814cb8be4224971a4505b5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/0eb2dbe49f814cb8be4224971a4505b5 2024-11-13T05:51:29,309 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into 0eb2dbe49f814cb8be4224971a4505b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:29,309 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:29,309 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=12, startTime=1731477089250; duration=0sec 2024-11-13T05:51:29,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:29,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:29,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:29,311 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:29,311 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:29,311 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:29,312 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/115520e507de48d5a63c544d645c270b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4b78d436c3e480fb21e889b57088179, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/715fb1f81c004d94982b4839a41b6bb2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/6efb87d8b60f4088b7818007f4a7f6a0] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=47.5 K 2024-11-13T05:51:29,312 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 115520e507de48d5a63c544d645c270b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731477082712 2024-11-13T05:51:29,313 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4b78d436c3e480fb21e889b57088179, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731477084883 2024-11-13T05:51:29,314 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 715fb1f81c004d94982b4839a41b6bb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731477086057 2024-11-13T05:51:29,314 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6efb87d8b60f4088b7818007f4a7f6a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477088269 2024-11-13T05:51:29,322 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:29,322 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b3912b0b9f2d4936b93e8172acbfefcc is 50, key is test_row_0/C:col10/1731477088277/Put/seqid=0 2024-11-13T05:51:29,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742107_1283 (size=12493) 2024-11-13T05:51:29,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742108_1284 (size=12493) 2024-11-13T05:51:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:29,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:51:29,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:29,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:29,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:29,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:29,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/bd75e591bdea41888c2a8a937be08889 is 50, key is test_row_0/A:col10/1731477089562/Put/seqid=0 2024-11-13T05:51:29,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742109_1285 (size=16931) 2024-11-13T05:51:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477149580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477149581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477149582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477149585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477149586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477149691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477149691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477149691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477149701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477149702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,736 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/24ac1f56cda8444098b95877d6f8015c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/24ac1f56cda8444098b95877d6f8015c 2024-11-13T05:51:29,741 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into 24ac1f56cda8444098b95877d6f8015c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:29,741 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:29,741 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=12, startTime=1731477089250; duration=0sec 2024-11-13T05:51:29,741 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:29,741 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:29,745 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b3912b0b9f2d4936b93e8172acbfefcc as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b3912b0b9f2d4936b93e8172acbfefcc 2024-11-13T05:51:29,750 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into b3912b0b9f2d4936b93e8172acbfefcc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:29,750 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:29,750 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=12, startTime=1731477089250; duration=0sec 2024-11-13T05:51:29,750 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:29,750 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:29,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477149901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477149901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477149901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477149912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:29,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477149912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:29,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T05:51:29,941 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:29,942 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T05:51:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T05:51:29,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:29,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:29,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:29,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/bd75e591bdea41888c2a8a937be08889 2024-11-13T05:51:29,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/2dddada9bc5e4908ae3ffa1a49e08b53 is 50, key is test_row_0/B:col10/1731477089562/Put/seqid=0 2024-11-13T05:51:29,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742110_1286 (size=12151) 2024-11-13T05:51:29,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/2dddada9bc5e4908ae3ffa1a49e08b53 2024-11-13T05:51:29,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/dd5f3eb9b1b84e8b89b2323016cc771c is 50, key is test_row_0/C:col10/1731477089562/Put/seqid=0 2024-11-13T05:51:30,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742111_1287 (size=12151) 2024-11-13T05:51:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T05:51:30,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T05:51:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477150211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477150211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477150211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477150222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477150222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T05:51:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T05:51:30,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T05:51:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:30,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/dd5f3eb9b1b84e8b89b2323016cc771c 2024-11-13T05:51:30,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/bd75e591bdea41888c2a8a937be08889 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/bd75e591bdea41888c2a8a937be08889 2024-11-13T05:51:30,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/bd75e591bdea41888c2a8a937be08889, entries=250, sequenceid=195, filesize=16.5 K 2024-11-13T05:51:30,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/2dddada9bc5e4908ae3ffa1a49e08b53 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2dddada9bc5e4908ae3ffa1a49e08b53 2024-11-13T05:51:30,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2dddada9bc5e4908ae3ffa1a49e08b53, entries=150, sequenceid=195, filesize=11.9 K 2024-11-13T05:51:30,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/dd5f3eb9b1b84e8b89b2323016cc771c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd5f3eb9b1b84e8b89b2323016cc771c 2024-11-13T05:51:30,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd5f3eb9b1b84e8b89b2323016cc771c, entries=150, sequenceid=195, filesize=11.9 K 2024-11-13T05:51:30,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f3802ebb1be649f045361204f3bf7507 in 893ms, sequenceid=195, compaction requested=false 2024-11-13T05:51:30,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:30,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T05:51:30,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:30,554 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T05:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:30,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:30,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/734b99a1dac64a64a92d0bbef1848b48 is 50, key is test_row_0/A:col10/1731477089581/Put/seqid=0 2024-11-13T05:51:30,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742112_1288 (size=12151) 2024-11-13T05:51:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T05:51:30,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477150736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477150736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477150737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477150738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477150738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477150851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477150851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477150851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477150851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477150851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:30,963 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/734b99a1dac64a64a92d0bbef1848b48 2024-11-13T05:51:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/82781f0a4d6644708c5e0c6e413bbcb3 is 50, key is test_row_0/B:col10/1731477089581/Put/seqid=0 2024-11-13T05:51:30,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742113_1289 (size=12151) 2024-11-13T05:51:31,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477151061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T05:51:31,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477151371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477151371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477151371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477151371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477151371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,380 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/82781f0a4d6644708c5e0c6e413bbcb3 2024-11-13T05:51:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/77d90afdc0524db7936129eee01f4900 is 50, key is test_row_0/C:col10/1731477089581/Put/seqid=0 2024-11-13T05:51:31,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742114_1290 (size=12151) 2024-11-13T05:51:31,390 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/77d90afdc0524db7936129eee01f4900 2024-11-13T05:51:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/734b99a1dac64a64a92d0bbef1848b48 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/734b99a1dac64a64a92d0bbef1848b48 2024-11-13T05:51:31,397 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/734b99a1dac64a64a92d0bbef1848b48, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T05:51:31,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/82781f0a4d6644708c5e0c6e413bbcb3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/82781f0a4d6644708c5e0c6e413bbcb3 2024-11-13T05:51:31,401 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/82781f0a4d6644708c5e0c6e413bbcb3, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T05:51:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/77d90afdc0524db7936129eee01f4900 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/77d90afdc0524db7936129eee01f4900 2024-11-13T05:51:31,405 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/77d90afdc0524db7936129eee01f4900, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T05:51:31,406 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f3802ebb1be649f045361204f3bf7507 in 851ms, sequenceid=215, compaction requested=true 2024-11-13T05:51:31,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:31,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:31,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T05:51:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T05:51:31,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T05:51:31,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-13T05:51:31,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4670 sec 2024-11-13T05:51:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:31,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:51:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:31,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/1b576c242c084b75ba4e6d120e3c0318 is 50, key is test_row_0/A:col10/1731477091885/Put/seqid=0 2024-11-13T05:51:31,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742115_1291 (size=12151) 2024-11-13T05:51:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477151908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477151909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477151910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477151910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477151911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477152021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477152021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477152022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477152022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477152022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T05:51:32,092 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:32,092 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T05:51:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:51:32,094 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:32,094 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:32,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:51:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477152232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477152232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477152232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477152232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477152232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/1b576c242c084b75ba4e6d120e3c0318 2024-11-13T05:51:32,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/b4b9cca6878b484bbfbfb61f7119a802 is 50, key is test_row_0/B:col10/1731477091885/Put/seqid=0 2024-11-13T05:51:32,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742116_1292 (size=12151) 2024-11-13T05:51:32,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:51:32,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477152542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477152542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477152542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477152542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477152542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:32,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:32,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:32,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:32,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/b4b9cca6878b484bbfbfb61f7119a802 2024-11-13T05:51:32,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/d0673a13857b4b0ba7817afb71cd155b is 50, key is test_row_0/C:col10/1731477091885/Put/seqid=0 2024-11-13T05:51:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:51:32,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742117_1293 (size=12151) 2024-11-13T05:51:32,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:32,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:33,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:33,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:33,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477153051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:33,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477153051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477153051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477153051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477153051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:33,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/d0673a13857b4b0ba7817afb71cd155b 2024-11-13T05:51:33,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/1b576c242c084b75ba4e6d120e3c0318 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1b576c242c084b75ba4e6d120e3c0318 2024-11-13T05:51:33,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1b576c242c084b75ba4e6d120e3c0318, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T05:51:33,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/b4b9cca6878b484bbfbfb61f7119a802 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b4b9cca6878b484bbfbfb61f7119a802 2024-11-13T05:51:33,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b4b9cca6878b484bbfbfb61f7119a802, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T05:51:33,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/d0673a13857b4b0ba7817afb71cd155b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d0673a13857b4b0ba7817afb71cd155b 2024-11-13T05:51:33,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d0673a13857b4b0ba7817afb71cd155b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T05:51:33,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f3802ebb1be649f045361204f3bf7507 in 1256ms, sequenceid=233, compaction requested=true 2024-11-13T05:51:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:33,141 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:33,141 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:33,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:33,142 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:33,142 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:33,143 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:33,143 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/24ac1f56cda8444098b95877d6f8015c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2dddada9bc5e4908ae3ffa1a49e08b53, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/82781f0a4d6644708c5e0c6e413bbcb3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b4b9cca6878b484bbfbfb61f7119a802] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=47.8 K 2024-11-13T05:51:33,143 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,143 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/0eb2dbe49f814cb8be4224971a4505b5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/bd75e591bdea41888c2a8a937be08889, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/734b99a1dac64a64a92d0bbef1848b48, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1b576c242c084b75ba4e6d120e3c0318] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=52.5 K 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ac1f56cda8444098b95877d6f8015c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477088269 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb2dbe49f814cb8be4224971a4505b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477088269 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dddada9bc5e4908ae3ffa1a49e08b53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731477088413 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd75e591bdea41888c2a8a937be08889, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731477088412 2024-11-13T05:51:33,143 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 82781f0a4d6644708c5e0c6e413bbcb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731477089581 2024-11-13T05:51:33,144 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b9cca6878b484bbfbfb61f7119a802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477091885 2024-11-13T05:51:33,144 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734b99a1dac64a64a92d0bbef1848b48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731477089581 2024-11-13T05:51:33,144 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b576c242c084b75ba4e6d120e3c0318, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477091885 2024-11-13T05:51:33,153 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:33,153 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:33,154 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/9db059423ede4a639e7c0b59ff6d54c9 is 50, key is test_row_0/B:col10/1731477091885/Put/seqid=0 2024-11-13T05:51:33,154 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/96ef2e8e91554c4d8eb354d3cfa43234 is 50, key is test_row_0/A:col10/1731477091885/Put/seqid=0 2024-11-13T05:51:33,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T05:51:33,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,180 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:51:33,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:33,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742118_1294 (size=12629) 2024-11-13T05:51:33,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742119_1295 (size=12629) 2024-11-13T05:51:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/7312dc41200543c7b3ebcbfeb891e670 is 50, key is test_row_0/A:col10/1731477091906/Put/seqid=0 2024-11-13T05:51:33,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742120_1296 (size=12151) 2024-11-13T05:51:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:51:33,587 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/9db059423ede4a639e7c0b59ff6d54c9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/9db059423ede4a639e7c0b59ff6d54c9 2024-11-13T05:51:33,588 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/96ef2e8e91554c4d8eb354d3cfa43234 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/96ef2e8e91554c4d8eb354d3cfa43234 2024-11-13T05:51:33,592 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into 9db059423ede4a639e7c0b59ff6d54c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:33,592 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:33,592 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=12, startTime=1731477093141; duration=0sec 2024-11-13T05:51:33,592 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:33,592 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:33,592 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:51:33,593 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into 96ef2e8e91554c4d8eb354d3cfa43234(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:33,593 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:33,593 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=12, startTime=1731477093141; duration=0sec 2024-11-13T05:51:33,593 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:33,593 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:33,594 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:51:33,594 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:33,594 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,594 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b3912b0b9f2d4936b93e8172acbfefcc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd5f3eb9b1b84e8b89b2323016cc771c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/77d90afdc0524db7936129eee01f4900, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d0673a13857b4b0ba7817afb71cd155b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=47.8 K 2024-11-13T05:51:33,594 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b3912b0b9f2d4936b93e8172acbfefcc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731477088269 2024-11-13T05:51:33,594 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting dd5f3eb9b1b84e8b89b2323016cc771c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731477088413 2024-11-13T05:51:33,595 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d90afdc0524db7936129eee01f4900, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731477089581 2024-11-13T05:51:33,595 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d0673a13857b4b0ba7817afb71cd155b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477091885 2024-11-13T05:51:33,603 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/7312dc41200543c7b3ebcbfeb891e670 2024-11-13T05:51:33,604 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:33,604 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/a0cda591888d4a4fa0dc08ad90339b0a is 50, key is test_row_0/C:col10/1731477091885/Put/seqid=0 2024-11-13T05:51:33,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742121_1297 (size=12629) 2024-11-13T05:51:33,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/c73c23edd1ca46bea0e2156c5badb9f0 is 50, key is test_row_0/B:col10/1731477091906/Put/seqid=0 2024-11-13T05:51:33,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742122_1298 (size=12151) 2024-11-13T05:51:33,626 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/c73c23edd1ca46bea0e2156c5badb9f0 2024-11-13T05:51:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/e8b856a70e224ad0901c3b314e195d65 is 50, key is test_row_0/C:col10/1731477091906/Put/seqid=0 2024-11-13T05:51:33,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742123_1299 (size=12151) 2024-11-13T05:51:33,636 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/e8b856a70e224ad0901c3b314e195d65 2024-11-13T05:51:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/7312dc41200543c7b3ebcbfeb891e670 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7312dc41200543c7b3ebcbfeb891e670 2024-11-13T05:51:33,645 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7312dc41200543c7b3ebcbfeb891e670, entries=150, sequenceid=247, filesize=11.9 K 2024-11-13T05:51:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/c73c23edd1ca46bea0e2156c5badb9f0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/c73c23edd1ca46bea0e2156c5badb9f0 2024-11-13T05:51:33,649 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/c73c23edd1ca46bea0e2156c5badb9f0, entries=150, sequenceid=247, filesize=11.9 K 2024-11-13T05:51:33,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/e8b856a70e224ad0901c3b314e195d65 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/e8b856a70e224ad0901c3b314e195d65 2024-11-13T05:51:33,654 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/e8b856a70e224ad0901c3b314e195d65, entries=150, sequenceid=247, filesize=11.9 K 2024-11-13T05:51:33,655 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for f3802ebb1be649f045361204f3bf7507 in 493ms, sequenceid=247, compaction requested=false 2024-11-13T05:51:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T05:51:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T05:51:33,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T05:51:33,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5620 sec 2024-11-13T05:51:33,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5660 sec 2024-11-13T05:51:34,018 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/a0cda591888d4a4fa0dc08ad90339b0a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/a0cda591888d4a4fa0dc08ad90339b0a 2024-11-13T05:51:34,022 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into a0cda591888d4a4fa0dc08ad90339b0a(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:34,022 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:34,022 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=12, startTime=1731477093141; duration=0sec 2024-11-13T05:51:34,022 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:34,022 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:34,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:51:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:34,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:34,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c96c21db9ff94fc7885a3bc9392beee4 is 50, key is test_row_0/A:col10/1731477094063/Put/seqid=0 2024-11-13T05:51:34,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477154088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477094080, max timestamp=1731477094092Num uniques 859; 2024-11-13T05:51:34,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:34,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477094080, max timestamp=1731477094092Num uniques 859; 2024-11-13T05:51:34,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477094080, max timestamp=1731477094092Num uniques 859; 2024-11-13T05:51:34,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:34,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742124_1300 (size=17131) 2024-11-13T05:51:34,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477154117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477154118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477154118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477154119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477154201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477154232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477154232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477154232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477154232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T05:51:34,242 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:34,242 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-13T05:51:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T05:51:34,243 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:34,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:34,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T05:51:34,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T05:51:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:34,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477154412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477154441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477154441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477154441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477154441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c96c21db9ff94fc7885a3bc9392beee4 2024-11-13T05:51:34,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/73a4a986a4504f51a5e7561260eab6f5 is 50, key is test_row_0/B:col10/1731477094063/Put/seqid=0 2024-11-13T05:51:34,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742125_1301 (size=12251) 2024-11-13T05:51:34,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/73a4a986a4504f51a5e7561260eab6f5 2024-11-13T05:51:34,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b4ee5bc71bc842409af24e4d07556665 is 50, key is test_row_0/C:col10/1731477094063/Put/seqid=0 2024-11-13T05:51:34,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742126_1302 (size=12251) 2024-11-13T05:51:34,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T05:51:34,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:34,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T05:51:34,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T05:51:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477154722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477154751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477154751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477154751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477154751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:34,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T05:51:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T05:51:34,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b4ee5bc71bc842409af24e4d07556665 2024-11-13T05:51:34,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c96c21db9ff94fc7885a3bc9392beee4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c96c21db9ff94fc7885a3bc9392beee4 2024-11-13T05:51:34,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c96c21db9ff94fc7885a3bc9392beee4, entries=250, sequenceid=261, filesize=16.7 K 2024-11-13T05:51:34,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/73a4a986a4504f51a5e7561260eab6f5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/73a4a986a4504f51a5e7561260eab6f5 2024-11-13T05:51:34,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/73a4a986a4504f51a5e7561260eab6f5, entries=150, sequenceid=261, filesize=12.0 K 2024-11-13T05:51:34,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/b4ee5bc71bc842409af24e4d07556665 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4ee5bc71bc842409af24e4d07556665 2024-11-13T05:51:34,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4ee5bc71bc842409af24e4d07556665, entries=150, sequenceid=261, filesize=12.0 K 2024-11-13T05:51:34,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f3802ebb1be649f045361204f3bf7507 in 872ms, sequenceid=261, compaction requested=true 2024-11-13T05:51:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:34,941 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:34,941 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:34,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:34,941 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:34,941 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:34,941 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:34,941 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:34,942 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,942 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,942 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/96ef2e8e91554c4d8eb354d3cfa43234, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7312dc41200543c7b3ebcbfeb891e670, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c96c21db9ff94fc7885a3bc9392beee4] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=40.9 K 2024-11-13T05:51:34,942 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/9db059423ede4a639e7c0b59ff6d54c9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/c73c23edd1ca46bea0e2156c5badb9f0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/73a4a986a4504f51a5e7561260eab6f5] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.2 K 2024-11-13T05:51:34,942 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ef2e8e91554c4d8eb354d3cfa43234, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477091885 2024-11-13T05:51:34,942 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db059423ede4a639e7c0b59ff6d54c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477091885 2024-11-13T05:51:34,942 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7312dc41200543c7b3ebcbfeb891e670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731477091906 2024-11-13T05:51:34,942 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting c73c23edd1ca46bea0e2156c5badb9f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731477091906 2024-11-13T05:51:34,942 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c96c21db9ff94fc7885a3bc9392beee4, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477094063 2024-11-13T05:51:34,942 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a4a986a4504f51a5e7561260eab6f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477094063 2024-11-13T05:51:34,949 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:34,949 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:34,950 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/75ee8da83aeb4a9d868bb9dd254f4170 is 50, key is test_row_0/B:col10/1731477094063/Put/seqid=0 2024-11-13T05:51:34,950 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/7949e027d1e84b9da6f847bc99c4e5fc is 50, key is test_row_0/A:col10/1731477094063/Put/seqid=0 2024-11-13T05:51:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742128_1304 (size=12831) 2024-11-13T05:51:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742127_1303 (size=12831) 2024-11-13T05:51:34,959 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/7949e027d1e84b9da6f847bc99c4e5fc as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7949e027d1e84b9da6f847bc99c4e5fc 2024-11-13T05:51:34,964 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into 7949e027d1e84b9da6f847bc99c4e5fc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:34,964 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:34,965 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=13, startTime=1731477094941; duration=0sec 2024-11-13T05:51:34,965 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:34,965 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:34,965 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:34,966 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:34,966 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:34,966 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:34,966 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/a0cda591888d4a4fa0dc08ad90339b0a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/e8b856a70e224ad0901c3b314e195d65, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4ee5bc71bc842409af24e4d07556665] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.2 K 2024-11-13T05:51:34,966 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0cda591888d4a4fa0dc08ad90339b0a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477091885 2024-11-13T05:51:34,967 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b856a70e224ad0901c3b314e195d65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731477091906 2024-11-13T05:51:34,967 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4ee5bc71bc842409af24e4d07556665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477094063 2024-11-13T05:51:34,974 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:34,974 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/516b8c6e5c704443929bc40486c4545e is 50, key is test_row_0/C:col10/1731477094063/Put/seqid=0 2024-11-13T05:51:34,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742129_1305 (size=12831) 2024-11-13T05:51:34,988 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/516b8c6e5c704443929bc40486c4545e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/516b8c6e5c704443929bc40486c4545e 2024-11-13T05:51:34,992 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into 516b8c6e5c704443929bc40486c4545e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:34,993 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:34,993 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=13, startTime=1731477094941; duration=0sec 2024-11-13T05:51:34,993 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:34,993 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:35,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T05:51:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:35,008 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T05:51:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:35,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:35,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f702405bc6ea4763b193189e7a9fc8ca is 50, key is test_row_0/A:col10/1731477094118/Put/seqid=0 2024-11-13T05:51:35,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742130_1306 (size=12301) 2024-11-13T05:51:35,035 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f702405bc6ea4763b193189e7a9fc8ca 2024-11-13T05:51:35,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/d7fac002150f42cd8f02bc5b52922cae is 50, key is test_row_0/B:col10/1731477094118/Put/seqid=0 2024-11-13T05:51:35,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742131_1307 (size=12301) 2024-11-13T05:51:35,057 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/d7fac002150f42cd8f02bc5b52922cae 2024-11-13T05:51:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/3fc080db79c0423b830c6a8e3193742d is 50, key is test_row_0/C:col10/1731477094118/Put/seqid=0 2024-11-13T05:51:35,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742132_1308 (size=12301) 2024-11-13T05:51:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:35,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:35,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477155262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477155262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477155265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477155266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477155267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,359 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/75ee8da83aeb4a9d868bb9dd254f4170 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/75ee8da83aeb4a9d868bb9dd254f4170 2024-11-13T05:51:35,362 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into 75ee8da83aeb4a9d868bb9dd254f4170(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:35,362 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:35,362 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=13, startTime=1731477094941; duration=0sec 2024-11-13T05:51:35,363 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:35,363 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:35,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477155372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T05:51:35,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477155381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477155381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,469 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/3fc080db79c0423b830c6a8e3193742d 2024-11-13T05:51:35,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f702405bc6ea4763b193189e7a9fc8ca as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f702405bc6ea4763b193189e7a9fc8ca 2024-11-13T05:51:35,476 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f702405bc6ea4763b193189e7a9fc8ca, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:51:35,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/d7fac002150f42cd8f02bc5b52922cae as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d7fac002150f42cd8f02bc5b52922cae 2024-11-13T05:51:35,481 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d7fac002150f42cd8f02bc5b52922cae, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:51:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/3fc080db79c0423b830c6a8e3193742d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/3fc080db79c0423b830c6a8e3193742d 2024-11-13T05:51:35,485 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/3fc080db79c0423b830c6a8e3193742d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:51:35,486 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for f3802ebb1be649f045361204f3bf7507 in 478ms, sequenceid=298, compaction requested=false 2024-11-13T05:51:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-13T05:51:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-13T05:51:35,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T05:51:35,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2430 sec 2024-11-13T05:51:35,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.2480 sec 2024-11-13T05:51:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:35,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T05:51:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:35,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:35,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/b337da597b704d5e8675b7d99eb73cec is 50, key is test_row_0/A:col10/1731477095583/Put/seqid=0 2024-11-13T05:51:35,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742133_1309 (size=14741) 2024-11-13T05:51:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477155626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477155626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477155627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477155741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477155741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477155741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477155952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477155952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477155952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:35,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/b337da597b704d5e8675b7d99eb73cec 2024-11-13T05:51:36,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/4343648bbc6f4832a5b42b55e058d053 is 50, key is test_row_0/B:col10/1731477095583/Put/seqid=0 2024-11-13T05:51:36,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742134_1310 (size=12301) 2024-11-13T05:51:36,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477156261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477156262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477156262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477156281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477156281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,283 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:36,283 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T05:51:36,391 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:36,392 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-13T05:51:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:51:36,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:36,394 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:36,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:36,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/4343648bbc6f4832a5b42b55e058d053 2024-11-13T05:51:36,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/9d9f39fd78ad4b3b8b6474144968ac27 is 50, key is test_row_0/C:col10/1731477095583/Put/seqid=0 2024-11-13T05:51:36,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742135_1311 (size=12301) 2024-11-13T05:51:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:51:36,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T05:51:36,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:36,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:36,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:36,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T05:51:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:51:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477156771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477156771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477156771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:36,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/9d9f39fd78ad4b3b8b6474144968ac27 2024-11-13T05:51:36,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/b337da597b704d5e8675b7d99eb73cec as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b337da597b704d5e8675b7d99eb73cec 2024-11-13T05:51:36,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b337da597b704d5e8675b7d99eb73cec, entries=200, sequenceid=311, filesize=14.4 K 2024-11-13T05:51:36,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/4343648bbc6f4832a5b42b55e058d053 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/4343648bbc6f4832a5b42b55e058d053 2024-11-13T05:51:36,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/4343648bbc6f4832a5b42b55e058d053, entries=150, sequenceid=311, filesize=12.0 K 2024-11-13T05:51:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/9d9f39fd78ad4b3b8b6474144968ac27 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9d9f39fd78ad4b3b8b6474144968ac27 2024-11-13T05:51:36,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9d9f39fd78ad4b3b8b6474144968ac27, entries=150, sequenceid=311, filesize=12.0 K 2024-11-13T05:51:36,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f3802ebb1be649f045361204f3bf7507 in 1252ms, sequenceid=311, compaction requested=true 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:36,836 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:36,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:36,836 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:36,837 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:36,837 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:36,837 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:36,837 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:36,837 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,837 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,837 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7949e027d1e84b9da6f847bc99c4e5fc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f702405bc6ea4763b193189e7a9fc8ca, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b337da597b704d5e8675b7d99eb73cec] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=38.9 K 2024-11-13T05:51:36,837 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/75ee8da83aeb4a9d868bb9dd254f4170, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d7fac002150f42cd8f02bc5b52922cae, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/4343648bbc6f4832a5b42b55e058d053] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.6 K 2024-11-13T05:51:36,837 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7949e027d1e84b9da6f847bc99c4e5fc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477094063 2024-11-13T05:51:36,837 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ee8da83aeb4a9d868bb9dd254f4170, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477094063 2024-11-13T05:51:36,838 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting f702405bc6ea4763b193189e7a9fc8ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477094102 2024-11-13T05:51:36,838 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d7fac002150f42cd8f02bc5b52922cae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477094102 2024-11-13T05:51:36,838 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b337da597b704d5e8675b7d99eb73cec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731477095247 2024-11-13T05:51:36,838 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4343648bbc6f4832a5b42b55e058d053, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731477095247 2024-11-13T05:51:36,852 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:36,852 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:36,852 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/f6cd53308342434eafceb86bda70f96a is 50, key is test_row_0/B:col10/1731477095583/Put/seqid=0 2024-11-13T05:51:36,852 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f717143fccc648278544613c101e98d7 is 50, key is test_row_0/A:col10/1731477095583/Put/seqid=0 2024-11-13T05:51:36,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T05:51:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,854 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:51:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:36,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/8596a35240c34573bfda5af9ab882fbc is 50, key is test_row_0/A:col10/1731477095613/Put/seqid=0 2024-11-13T05:51:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742137_1313 (size=12983) 2024-11-13T05:51:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742136_1312 (size=12983) 2024-11-13T05:51:36,868 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f717143fccc648278544613c101e98d7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f717143fccc648278544613c101e98d7 2024-11-13T05:51:36,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742138_1314 (size=12301) 2024-11-13T05:51:36,874 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into f717143fccc648278544613c101e98d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:36,874 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:36,874 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=13, startTime=1731477096836; duration=0sec 2024-11-13T05:51:36,874 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:36,874 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:36,874 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:36,875 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:36,875 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:36,875 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:36,875 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/516b8c6e5c704443929bc40486c4545e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/3fc080db79c0423b830c6a8e3193742d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9d9f39fd78ad4b3b8b6474144968ac27] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.6 K 2024-11-13T05:51:36,875 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516b8c6e5c704443929bc40486c4545e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477094063 2024-11-13T05:51:36,875 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc080db79c0423b830c6a8e3193742d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477094102 2024-11-13T05:51:36,875 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9f39fd78ad4b3b8b6474144968ac27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731477095247 2024-11-13T05:51:36,882 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:36,883 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1 is 50, key is test_row_0/C:col10/1731477095583/Put/seqid=0 2024-11-13T05:51:36,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742139_1315 (size=12983) 2024-11-13T05:51:36,893 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1 2024-11-13T05:51:36,898 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into 12bfd2ac1c0f42b5a00f48ae1a8dd7a1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:36,898 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:36,898 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=13, startTime=1731477096836; duration=0sec 2024-11-13T05:51:36,898 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:36,898 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:51:37,267 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/f6cd53308342434eafceb86bda70f96a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f6cd53308342434eafceb86bda70f96a 2024-11-13T05:51:37,269 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/8596a35240c34573bfda5af9ab882fbc 2024-11-13T05:51:37,271 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into f6cd53308342434eafceb86bda70f96a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:37,271 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:37,271 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=13, startTime=1731477096836; duration=0sec 2024-11-13T05:51:37,272 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:37,272 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/1c34657f11474080ba22ead3063d18be is 50, key is test_row_0/B:col10/1731477095613/Put/seqid=0 2024-11-13T05:51:37,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742140_1316 (size=12301) 2024-11-13T05:51:37,281 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/1c34657f11474080ba22ead3063d18be 2024-11-13T05:51:37,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/28630f86867c4dc5a8419b5b359a3f42 is 50, key is test_row_0/C:col10/1731477095613/Put/seqid=0 2024-11-13T05:51:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742141_1317 (size=12301) 2024-11-13T05:51:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:51:37,690 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/28630f86867c4dc5a8419b5b359a3f42 2024-11-13T05:51:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/8596a35240c34573bfda5af9ab882fbc as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/8596a35240c34573bfda5af9ab882fbc 2024-11-13T05:51:37,698 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/8596a35240c34573bfda5af9ab882fbc, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T05:51:37,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/1c34657f11474080ba22ead3063d18be as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/1c34657f11474080ba22ead3063d18be 2024-11-13T05:51:37,702 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/1c34657f11474080ba22ead3063d18be, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T05:51:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/28630f86867c4dc5a8419b5b359a3f42 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/28630f86867c4dc5a8419b5b359a3f42 2024-11-13T05:51:37,707 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/28630f86867c4dc5a8419b5b359a3f42, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T05:51:37,707 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for f3802ebb1be649f045361204f3bf7507 in 853ms, sequenceid=331, compaction requested=false 2024-11-13T05:51:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-13T05:51:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-13T05:51:37,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T05:51:37,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-11-13T05:51:37,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.3190 sec 2024-11-13T05:51:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:37,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:51:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:37,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:37,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/eae2be1b528f4459940ca5bbef91572b is 50, key is test_row_0/A:col10/1731477097784/Put/seqid=0 2024-11-13T05:51:37,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742142_1318 (size=14741) 2024-11-13T05:51:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477157821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:37,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477097801, max timestamp=1731477097825Num uniques 859; 2024-11-13T05:51:37,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477097801, max timestamp=1731477097825Num uniques 859; 2024-11-13T05:51:37,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:37,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:37,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477097801, max timestamp=1731477097825Num uniques 859; 2024-11-13T05:51:37,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:51:37,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477157860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:37,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477157865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:37,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477157932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477157972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477157972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477158141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477158181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477158181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/eae2be1b528f4459940ca5bbef91572b 2024-11-13T05:51:38,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/d24b26bfcb9f49c884cd7fb78342539e is 50, key is test_row_0/B:col10/1731477097784/Put/seqid=0 2024-11-13T05:51:38,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742143_1319 (size=12301) 2024-11-13T05:51:38,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/d24b26bfcb9f49c884cd7fb78342539e 2024-11-13T05:51:38,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/dd0ee2e7b52a463cb9d4094d0b6099dd is 50, key is test_row_0/C:col10/1731477097784/Put/seqid=0 2024-11-13T05:51:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742144_1320 (size=12301) 2024-11-13T05:51:38,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38598 deadline: 1731477158302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,307 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38628 deadline: 1731477158312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,315 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477158451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477158491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477158491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T05:51:38,541 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:38,542 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-13T05:51:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T05:51:38,543 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:38,544 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:38,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:38,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/dd0ee2e7b52a463cb9d4094d0b6099dd 2024-11-13T05:51:38,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/eae2be1b528f4459940ca5bbef91572b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/eae2be1b528f4459940ca5bbef91572b 2024-11-13T05:51:38,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/eae2be1b528f4459940ca5bbef91572b, entries=200, sequenceid=345, filesize=14.4 K 2024-11-13T05:51:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/d24b26bfcb9f49c884cd7fb78342539e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d24b26bfcb9f49c884cd7fb78342539e 2024-11-13T05:51:38,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d24b26bfcb9f49c884cd7fb78342539e, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:51:38,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/dd0ee2e7b52a463cb9d4094d0b6099dd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd0ee2e7b52a463cb9d4094d0b6099dd 2024-11-13T05:51:38,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd0ee2e7b52a463cb9d4094d0b6099dd, entries=150, sequenceid=345, filesize=12.0 K 2024-11-13T05:51:38,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f3802ebb1be649f045361204f3bf7507 in 848ms, sequenceid=345, compaction requested=true 2024-11-13T05:51:38,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:38,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:38,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:38,641 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:38,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:38,642 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:38,643 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:38,643 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:38,643 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:38,643 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:38,643 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:38,643 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f6cd53308342434eafceb86bda70f96a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/1c34657f11474080ba22ead3063d18be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d24b26bfcb9f49c884cd7fb78342539e] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.7 K 2024-11-13T05:51:38,643 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:38,643 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f717143fccc648278544613c101e98d7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/8596a35240c34573bfda5af9ab882fbc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/eae2be1b528f4459940ca5bbef91572b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=39.1 K 2024-11-13T05:51:38,644 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f6cd53308342434eafceb86bda70f96a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731477095247 2024-11-13T05:51:38,644 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting f717143fccc648278544613c101e98d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731477095247 2024-11-13T05:51:38,644 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c34657f11474080ba22ead3063d18be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731477095613 2024-11-13T05:51:38,644 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8596a35240c34573bfda5af9ab882fbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731477095613 2024-11-13T05:51:38,645 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d24b26bfcb9f49c884cd7fb78342539e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477097784 2024-11-13T05:51:38,645 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae2be1b528f4459940ca5bbef91572b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477097784 2024-11-13T05:51:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T05:51:38,655 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:38,656 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/a77c432255164e69bce5c51f2cd26ef5 is 50, key is test_row_0/A:col10/1731477097784/Put/seqid=0 2024-11-13T05:51:38,663 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:38,663 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/989c8fc493aa4ff29e5bd58ac86de779 is 50, key is test_row_0/B:col10/1731477097784/Put/seqid=0 2024-11-13T05:51:38,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-13T05:51:38,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:38,696 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-13T05:51:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:38,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:51:38,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742146_1322 (size=13085) 2024-11-13T05:51:38,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742145_1321 (size=13085) 2024-11-13T05:51:38,725 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/a77c432255164e69bce5c51f2cd26ef5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a77c432255164e69bce5c51f2cd26ef5 2024-11-13T05:51:38,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/470ad77c8ca94ca08bd21b0539f8e9da is 50, key is test_row_0/A:col10/1731477097850/Put/seqid=0 2024-11-13T05:51:38,732 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into a77c432255164e69bce5c51f2cd26ef5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:38,732 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:38,732 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=13, startTime=1731477098641; duration=0sec 2024-11-13T05:51:38,732 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:38,732 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:38,732 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:38,735 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:38,735 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:38,735 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:38,735 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/28630f86867c4dc5a8419b5b359a3f42, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd0ee2e7b52a463cb9d4094d0b6099dd] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.7 K 2024-11-13T05:51:38,736 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12bfd2ac1c0f42b5a00f48ae1a8dd7a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731477095247 2024-11-13T05:51:38,736 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28630f86867c4dc5a8419b5b359a3f42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731477095613 2024-11-13T05:51:38,737 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0ee2e7b52a463cb9d4094d0b6099dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477097784 2024-11-13T05:51:38,753 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:38,753 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/893cd25b7085456ba107f1670efcce06 is 50, key is test_row_0/C:col10/1731477097784/Put/seqid=0 2024-11-13T05:51:38,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742147_1323 (size=12301) 2024-11-13T05:51:38,770 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/470ad77c8ca94ca08bd21b0539f8e9da 2024-11-13T05:51:38,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742148_1324 (size=13085) 2024-11-13T05:51:38,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/edabbe98bbb145a6b93ccbe226a55fc6 is 50, key is test_row_0/B:col10/1731477097850/Put/seqid=0 2024-11-13T05:51:38,787 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/893cd25b7085456ba107f1670efcce06 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/893cd25b7085456ba107f1670efcce06 2024-11-13T05:51:38,792 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into 893cd25b7085456ba107f1670efcce06(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:38,792 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:38,792 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=13, startTime=1731477098642; duration=0sec 2024-11-13T05:51:38,792 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:38,792 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:38,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742149_1325 (size=12301) 2024-11-13T05:51:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T05:51:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:38,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477158993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477159001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477159001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477159101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,112 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/989c8fc493aa4ff29e5bd58ac86de779 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/989c8fc493aa4ff29e5bd58ac86de779 2024-11-13T05:51:39,116 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into 989c8fc493aa4ff29e5bd58ac86de779(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:39,116 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:39,116 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=13, startTime=1731477098641; duration=0sec 2024-11-13T05:51:39,117 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:39,117 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T05:51:39,196 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/edabbe98bbb145a6b93ccbe226a55fc6 2024-11-13T05:51:39,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/c76cd6bf571b43dc83bb8c321eaa8cde is 50, key is test_row_0/C:col10/1731477097850/Put/seqid=0 2024-11-13T05:51:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742150_1326 (size=12301) 2024-11-13T05:51:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477159312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,607 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/c76cd6bf571b43dc83bb8c321eaa8cde 2024-11-13T05:51:39,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/470ad77c8ca94ca08bd21b0539f8e9da as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/470ad77c8ca94ca08bd21b0539f8e9da 2024-11-13T05:51:39,614 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/470ad77c8ca94ca08bd21b0539f8e9da, entries=150, sequenceid=380, filesize=12.0 K 2024-11-13T05:51:39,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/edabbe98bbb145a6b93ccbe226a55fc6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/edabbe98bbb145a6b93ccbe226a55fc6 2024-11-13T05:51:39,617 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/edabbe98bbb145a6b93ccbe226a55fc6, entries=150, sequenceid=380, filesize=12.0 K 2024-11-13T05:51:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/c76cd6bf571b43dc83bb8c321eaa8cde as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/c76cd6bf571b43dc83bb8c321eaa8cde 2024-11-13T05:51:39,621 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/c76cd6bf571b43dc83bb8c321eaa8cde, entries=150, sequenceid=380, filesize=12.0 K 2024-11-13T05:51:39,622 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for f3802ebb1be649f045361204f3bf7507 in 925ms, sequenceid=380, compaction requested=false 2024-11-13T05:51:39,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:39,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:39,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-13T05:51:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-13T05:51:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:39,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T05:51:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:39,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T05:51:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:39,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0790 sec 2024-11-13T05:51:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:39,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/1725af8db5214c318a27a8a80f70dd59 is 50, key is test_row_0/A:col10/1731477098976/Put/seqid=0 2024-11-13T05:51:39,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742151_1327 (size=14741) 2024-11-13T05:51:39,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0890 sec 2024-11-13T05:51:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T05:51:39,682 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:39,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-13T05:51:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:39,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:39,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:39,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:39,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477159707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477159821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:39,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:39,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:39,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:39,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:39,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:40,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38614 deadline: 1731477160011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:40,014 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38660 deadline: 1731477160022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:40,025 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in f3802ebb1be649f045361204f3bf7507 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:40,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/1725af8db5214c318a27a8a80f70dd59 2024-11-13T05:51:40,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477160031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:40,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/2122e81f6e084318bbcf36239757de3e is 50, key is test_row_0/B:col10/1731477098976/Put/seqid=0 2024-11-13T05:51:40,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742152_1328 (size=12301) 2024-11-13T05:51:40,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:40,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:40,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477160341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:40,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/2122e81f6e084318bbcf36239757de3e 2024-11-13T05:51:40,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/414ce880f19d4435a89beb94ddc28ade is 50, key is test_row_0/C:col10/1731477098976/Put/seqid=0 2024-11-13T05:51:40,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:40,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:40,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742153_1329 (size=12301) 2024-11-13T05:51:40,466 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-13T05:51:40,466 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:40,466 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,466 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:40,466 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-13T05:51:40,466 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:40,466 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,466 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:40,468 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-13T05:51:40,468 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:40,469 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,469 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:40,469 INFO [Thread-1173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1173. 2024-11-13T05:51:40,470 DEBUG [Thread-1173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:40,470 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,470 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:40,471 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-13T05:51:40,471 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:40,471 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,471 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:40,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:40,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:40,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:40,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:40,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:40,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:40,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38568 deadline: 1731477160852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:51:40,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/414ce880f19d4435a89beb94ddc28ade 2024-11-13T05:51:40,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/1725af8db5214c318a27a8a80f70dd59 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1725af8db5214c318a27a8a80f70dd59 2024-11-13T05:51:40,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1725af8db5214c318a27a8a80f70dd59, entries=200, sequenceid=395, filesize=14.4 K 2024-11-13T05:51:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/2122e81f6e084318bbcf36239757de3e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2122e81f6e084318bbcf36239757de3e 2024-11-13T05:51:40,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2122e81f6e084318bbcf36239757de3e, entries=150, sequenceid=395, filesize=12.0 K 2024-11-13T05:51:40,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/414ce880f19d4435a89beb94ddc28ade as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/414ce880f19d4435a89beb94ddc28ade 2024-11-13T05:51:40,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/414ce880f19d4435a89beb94ddc28ade, entries=150, sequenceid=395, filesize=12.0 K 2024-11-13T05:51:40,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f3802ebb1be649f045361204f3bf7507 in 1248ms, sequenceid=395, compaction requested=true 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:40,872 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3802ebb1be649f045361204f3bf7507:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:40,872 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:40,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:40,873 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:40,873 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:40,873 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/B is initiating minor compaction (all files) 2024-11-13T05:51:40,873 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/A is initiating minor compaction (all files) 2024-11-13T05:51:40,873 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/B in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,873 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/A in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,873 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/989c8fc493aa4ff29e5bd58ac86de779, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/edabbe98bbb145a6b93ccbe226a55fc6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2122e81f6e084318bbcf36239757de3e] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.8 K 2024-11-13T05:51:40,873 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a77c432255164e69bce5c51f2cd26ef5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/470ad77c8ca94ca08bd21b0539f8e9da, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1725af8db5214c318a27a8a80f70dd59] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=39.2 K 2024-11-13T05:51:40,873 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 989c8fc493aa4ff29e5bd58ac86de779, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477097784 2024-11-13T05:51:40,873 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77c432255164e69bce5c51f2cd26ef5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477097784 2024-11-13T05:51:40,874 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting edabbe98bbb145a6b93ccbe226a55fc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731477097850 2024-11-13T05:51:40,874 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 470ad77c8ca94ca08bd21b0539f8e9da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731477097850 2024-11-13T05:51:40,874 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2122e81f6e084318bbcf36239757de3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731477098976 2024-11-13T05:51:40,874 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1725af8db5214c318a27a8a80f70dd59, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731477098976 2024-11-13T05:51:40,881 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:40,881 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#A#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:40,881 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/61d86206886e46a7b134a44fbbd6360c is 50, key is test_row_0/A:col10/1731477098976/Put/seqid=0 2024-11-13T05:51:40,881 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/ff1a961e548346c896a67685eb3f125f is 50, key is test_row_0/B:col10/1731477098976/Put/seqid=0 2024-11-13T05:51:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742155_1331 (size=13187) 2024-11-13T05:51:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742154_1330 (size=13187) 2024-11-13T05:51:40,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:40,916 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c98911f9149d48e69734f5186e62f9e6 is 50, key is test_row_0/A:col10/1731477099706/Put/seqid=0 2024-11-13T05:51:40,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742156_1332 (size=12301) 2024-11-13T05:51:41,296 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/ff1a961e548346c896a67685eb3f125f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ff1a961e548346c896a67685eb3f125f 2024-11-13T05:51:41,296 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/61d86206886e46a7b134a44fbbd6360c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/61d86206886e46a7b134a44fbbd6360c 2024-11-13T05:51:41,301 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/A of f3802ebb1be649f045361204f3bf7507 into 61d86206886e46a7b134a44fbbd6360c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:41,301 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/B of f3802ebb1be649f045361204f3bf7507 into ff1a961e548346c896a67685eb3f125f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:41,301 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/B, priority=13, startTime=1731477100872; duration=0sec 2024-11-13T05:51:41,301 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/A, priority=13, startTime=1731477100872; duration=0sec 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:A 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:B 2024-11-13T05:51:41,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:41,302 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:41,302 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): f3802ebb1be649f045361204f3bf7507/C is initiating minor compaction (all files) 2024-11-13T05:51:41,302 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3802ebb1be649f045361204f3bf7507/C in TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:41,302 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/893cd25b7085456ba107f1670efcce06, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/c76cd6bf571b43dc83bb8c321eaa8cde, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/414ce880f19d4435a89beb94ddc28ade] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp, totalSize=36.8 K 2024-11-13T05:51:41,303 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 893cd25b7085456ba107f1670efcce06, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731477097784 2024-11-13T05:51:41,303 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c76cd6bf571b43dc83bb8c321eaa8cde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731477097850 2024-11-13T05:51:41,303 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 414ce880f19d4435a89beb94ddc28ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731477098976 2024-11-13T05:51:41,310 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3802ebb1be649f045361204f3bf7507#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:41,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/d02818bcec61408abd8c45e1ab6ece04 is 50, key is test_row_0/C:col10/1731477098976/Put/seqid=0 2024-11-13T05:51:41,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742157_1333 (size=13187) 2024-11-13T05:51:41,323 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c98911f9149d48e69734f5186e62f9e6 2024-11-13T05:51:41,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/56a6ade9022a43679a3ecaea9af1955b is 50, key is test_row_0/B:col10/1731477099706/Put/seqid=0 2024-11-13T05:51:41,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742158_1334 (size=12301) 2024-11-13T05:51:41,721 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/d02818bcec61408abd8c45e1ab6ece04 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d02818bcec61408abd8c45e1ab6ece04 2024-11-13T05:51:41,727 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3802ebb1be649f045361204f3bf7507/C of f3802ebb1be649f045361204f3bf7507 into d02818bcec61408abd8c45e1ab6ece04(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:41,727 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:41,727 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., storeName=f3802ebb1be649f045361204f3bf7507/C, priority=13, startTime=1731477100872; duration=0sec 2024-11-13T05:51:41,727 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:41,727 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3802ebb1be649f045361204f3bf7507:C 2024-11-13T05:51:41,736 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/56a6ade9022a43679a3ecaea9af1955b 2024-11-13T05:51:41,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/f59739f1230046a1b0b02c194b7e885a is 50, key is test_row_0/C:col10/1731477099706/Put/seqid=0 2024-11-13T05:51:41,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742159_1335 (size=12301) 2024-11-13T05:51:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:41,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. as already flushing 2024-11-13T05:51:41,863 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-13T05:51:41,863 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:41,863 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:41,863 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:41,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:42,044 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-13T05:51:42,044 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-13T05:51:42,044 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:42,044 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:42,044 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,044 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,044 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,044 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:42,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:42,155 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/f59739f1230046a1b0b02c194b7e885a 2024-11-13T05:51:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/c98911f9149d48e69734f5186e62f9e6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c98911f9149d48e69734f5186e62f9e6 2024-11-13T05:51:42,170 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c98911f9149d48e69734f5186e62f9e6, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T05:51:42,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/56a6ade9022a43679a3ecaea9af1955b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/56a6ade9022a43679a3ecaea9af1955b 2024-11-13T05:51:42,175 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/56a6ade9022a43679a3ecaea9af1955b, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T05:51:42,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/f59739f1230046a1b0b02c194b7e885a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f59739f1230046a1b0b02c194b7e885a 2024-11-13T05:51:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f59739f1230046a1b0b02c194b7e885a, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T05:51:42,181 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for f3802ebb1be649f045361204f3bf7507 in 1265ms, sequenceid=415, compaction requested=false 2024-11-13T05:51:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for f3802ebb1be649f045361204f3bf7507: 2024-11-13T05:51:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-13T05:51:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-13T05:51:42,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T05:51:42,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4980 sec 2024-11-13T05:51:42,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.5020 sec 2024-11-13T05:51:42,322 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-13T05:51:42,322 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:42,322 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,322 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:42,365 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-13T05:51:42,365 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:51:42,365 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,365 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:42,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-13T05:51:43,843 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3330 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3260 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3388 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3321 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3268 2024-11-13T05:51:43,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:51:43,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:51:43,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:51:43,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:43,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:43,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:51:43,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:51:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T05:51:43,853 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477103852"}]},"ts":"1731477103852"} 2024-11-13T05:51:43,855 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:51:43,855 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:51:43,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:51:43,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, UNASSIGN}] 2024-11-13T05:51:43,858 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, UNASSIGN 2024-11-13T05:51:43,858 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f3802ebb1be649f045361204f3bf7507, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:43,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, UNASSIGN because future has completed 2024-11-13T05:51:43,860 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:51:43,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:51:43,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T05:51:44,012 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:44,012 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:51:44,012 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing f3802ebb1be649f045361204f3bf7507, disabling compactions & flushes 2024-11-13T05:51:44,012 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:44,012 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:44,013 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. after waiting 0 ms 2024-11-13T05:51:44,013 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:44,013 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing f3802ebb1be649f045361204f3bf7507 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T05:51:44,013 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=A 2024-11-13T05:51:44,013 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:44,013 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=B 2024-11-13T05:51:44,013 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:44,014 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3802ebb1be649f045361204f3bf7507, store=C 2024-11-13T05:51:44,014 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:44,020 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f15514f5e1264e2b81ced599fe67d511 is 50, key is test_row_0/A:col10/1731477102321/Put/seqid=0 2024-11-13T05:51:44,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742160_1336 (size=12301) 2024-11-13T05:51:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T05:51:44,427 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f15514f5e1264e2b81ced599fe67d511 2024-11-13T05:51:44,437 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/e4322e59617a43b18496ac7ed91a32f3 is 50, key is test_row_0/B:col10/1731477102321/Put/seqid=0 2024-11-13T05:51:44,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742161_1337 (size=12301) 2024-11-13T05:51:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T05:51:44,495 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:51:44,841 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/e4322e59617a43b18496ac7ed91a32f3 2024-11-13T05:51:44,847 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/4b451a083fc64d4f97fc2009e603ab1c is 50, key is test_row_0/C:col10/1731477102321/Put/seqid=0 2024-11-13T05:51:44,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742162_1338 (size=12301) 2024-11-13T05:51:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T05:51:45,250 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/4b451a083fc64d4f97fc2009e603ab1c 2024-11-13T05:51:45,289 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/A/f15514f5e1264e2b81ced599fe67d511 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f15514f5e1264e2b81ced599fe67d511 2024-11-13T05:51:45,295 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f15514f5e1264e2b81ced599fe67d511, entries=150, sequenceid=426, filesize=12.0 K 2024-11-13T05:51:45,296 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/B/e4322e59617a43b18496ac7ed91a32f3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/e4322e59617a43b18496ac7ed91a32f3 2024-11-13T05:51:45,301 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/e4322e59617a43b18496ac7ed91a32f3, entries=150, sequenceid=426, filesize=12.0 K 2024-11-13T05:51:45,302 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/.tmp/C/4b451a083fc64d4f97fc2009e603ab1c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4b451a083fc64d4f97fc2009e603ab1c 2024-11-13T05:51:45,307 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4b451a083fc64d4f97fc2009e603ab1c, entries=150, sequenceid=426, filesize=12.0 K 2024-11-13T05:51:45,308 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f3802ebb1be649f045361204f3bf7507 in 1295ms, sequenceid=426, compaction requested=true 2024-11-13T05:51:45,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b97ee8e783c4ad2975ab1c152eb3e57, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a101445c95084b78997f964f779721da, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b218f581e5774adc97d6ff9342a40ec4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b88d699eeee4973bb8e30f0cc6c46a3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/e18c055660404e17abea3f971c85459c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/ee6d05f9e11c4179b1bff600f69a7cd4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c46a4a94a25f41f4bd7ab987d5b911d6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/5357b7555dfb4258a1c59ff63e57eab3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/001ea012a05d493385c6cf116c239a4a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/0eb2dbe49f814cb8be4224971a4505b5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2ea5e959317946b4aff6e2e78da1fd02, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/bd75e591bdea41888c2a8a937be08889, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/734b99a1dac64a64a92d0bbef1848b48, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/96ef2e8e91554c4d8eb354d3cfa43234, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1b576c242c084b75ba4e6d120e3c0318, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7312dc41200543c7b3ebcbfeb891e670, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c96c21db9ff94fc7885a3bc9392beee4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7949e027d1e84b9da6f847bc99c4e5fc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f702405bc6ea4763b193189e7a9fc8ca, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b337da597b704d5e8675b7d99eb73cec, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f717143fccc648278544613c101e98d7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/8596a35240c34573bfda5af9ab882fbc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/eae2be1b528f4459940ca5bbef91572b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a77c432255164e69bce5c51f2cd26ef5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/470ad77c8ca94ca08bd21b0539f8e9da, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1725af8db5214c318a27a8a80f70dd59] to archive 2024-11-13T05:51:45,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:51:45,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b97ee8e783c4ad2975ab1c152eb3e57 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b97ee8e783c4ad2975ab1c152eb3e57 2024-11-13T05:51:45,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a101445c95084b78997f964f779721da to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a101445c95084b78997f964f779721da 2024-11-13T05:51:45,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b218f581e5774adc97d6ff9342a40ec4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b218f581e5774adc97d6ff9342a40ec4 2024-11-13T05:51:45,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b88d699eeee4973bb8e30f0cc6c46a3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2b88d699eeee4973bb8e30f0cc6c46a3 2024-11-13T05:51:45,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/e18c055660404e17abea3f971c85459c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/e18c055660404e17abea3f971c85459c 2024-11-13T05:51:45,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/ee6d05f9e11c4179b1bff600f69a7cd4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/ee6d05f9e11c4179b1bff600f69a7cd4 2024-11-13T05:51:45,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c46a4a94a25f41f4bd7ab987d5b911d6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c46a4a94a25f41f4bd7ab987d5b911d6 2024-11-13T05:51:45,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/5357b7555dfb4258a1c59ff63e57eab3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/5357b7555dfb4258a1c59ff63e57eab3 2024-11-13T05:51:45,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/001ea012a05d493385c6cf116c239a4a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/001ea012a05d493385c6cf116c239a4a 2024-11-13T05:51:45,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/0eb2dbe49f814cb8be4224971a4505b5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/0eb2dbe49f814cb8be4224971a4505b5 2024-11-13T05:51:45,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2ea5e959317946b4aff6e2e78da1fd02 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/2ea5e959317946b4aff6e2e78da1fd02 2024-11-13T05:51:45,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/bd75e591bdea41888c2a8a937be08889 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/bd75e591bdea41888c2a8a937be08889 2024-11-13T05:51:45,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/734b99a1dac64a64a92d0bbef1848b48 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/734b99a1dac64a64a92d0bbef1848b48 2024-11-13T05:51:45,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/96ef2e8e91554c4d8eb354d3cfa43234 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/96ef2e8e91554c4d8eb354d3cfa43234 2024-11-13T05:51:45,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1b576c242c084b75ba4e6d120e3c0318 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1b576c242c084b75ba4e6d120e3c0318 2024-11-13T05:51:45,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7312dc41200543c7b3ebcbfeb891e670 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7312dc41200543c7b3ebcbfeb891e670 2024-11-13T05:51:45,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c96c21db9ff94fc7885a3bc9392beee4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c96c21db9ff94fc7885a3bc9392beee4 2024-11-13T05:51:45,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7949e027d1e84b9da6f847bc99c4e5fc to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/7949e027d1e84b9da6f847bc99c4e5fc 2024-11-13T05:51:45,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f702405bc6ea4763b193189e7a9fc8ca to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f702405bc6ea4763b193189e7a9fc8ca 2024-11-13T05:51:45,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b337da597b704d5e8675b7d99eb73cec to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/b337da597b704d5e8675b7d99eb73cec 2024-11-13T05:51:45,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f717143fccc648278544613c101e98d7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f717143fccc648278544613c101e98d7 2024-11-13T05:51:45,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/8596a35240c34573bfda5af9ab882fbc to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/8596a35240c34573bfda5af9ab882fbc 2024-11-13T05:51:45,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/eae2be1b528f4459940ca5bbef91572b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/eae2be1b528f4459940ca5bbef91572b 2024-11-13T05:51:45,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a77c432255164e69bce5c51f2cd26ef5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/a77c432255164e69bce5c51f2cd26ef5 2024-11-13T05:51:45,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/470ad77c8ca94ca08bd21b0539f8e9da to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/470ad77c8ca94ca08bd21b0539f8e9da 2024-11-13T05:51:45,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1725af8db5214c318a27a8a80f70dd59 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/1725af8db5214c318a27a8a80f70dd59 2024-11-13T05:51:45,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ea28fda89b494421b532b62dc821286d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3c517c0b7e194b36924df88ad1497118, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3203fe8e181c443fa2ff9e805f5b5db0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b37ea0871c5c44eca3c7e2c81cb69fad, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7129fb2297414d8da054240d2a4647ed, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7fd2d43d44bf4d419aae85a3464a93d8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/966fb601cbe949809819e5964ff73fb9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f2f634c177d142b1963fe9f4ee3a8a70, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/bc83be3e014242ac93f0ab08dac6e060, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/24ac1f56cda8444098b95877d6f8015c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/abb73ca8b9524211afcba33c092728fd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2dddada9bc5e4908ae3ffa1a49e08b53, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/82781f0a4d6644708c5e0c6e413bbcb3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/9db059423ede4a639e7c0b59ff6d54c9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b4b9cca6878b484bbfbfb61f7119a802, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/c73c23edd1ca46bea0e2156c5badb9f0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/75ee8da83aeb4a9d868bb9dd254f4170, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/73a4a986a4504f51a5e7561260eab6f5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d7fac002150f42cd8f02bc5b52922cae, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f6cd53308342434eafceb86bda70f96a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/4343648bbc6f4832a5b42b55e058d053, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/1c34657f11474080ba22ead3063d18be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/989c8fc493aa4ff29e5bd58ac86de779, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d24b26bfcb9f49c884cd7fb78342539e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/edabbe98bbb145a6b93ccbe226a55fc6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2122e81f6e084318bbcf36239757de3e] to archive 2024-11-13T05:51:45,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:51:45,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ea28fda89b494421b532b62dc821286d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ea28fda89b494421b532b62dc821286d 2024-11-13T05:51:45,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3c517c0b7e194b36924df88ad1497118 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3c517c0b7e194b36924df88ad1497118 2024-11-13T05:51:45,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3203fe8e181c443fa2ff9e805f5b5db0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/3203fe8e181c443fa2ff9e805f5b5db0 2024-11-13T05:51:45,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b37ea0871c5c44eca3c7e2c81cb69fad to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b37ea0871c5c44eca3c7e2c81cb69fad 2024-11-13T05:51:45,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7129fb2297414d8da054240d2a4647ed to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7129fb2297414d8da054240d2a4647ed 2024-11-13T05:51:45,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7fd2d43d44bf4d419aae85a3464a93d8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/7fd2d43d44bf4d419aae85a3464a93d8 2024-11-13T05:51:45,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/966fb601cbe949809819e5964ff73fb9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/966fb601cbe949809819e5964ff73fb9 2024-11-13T05:51:45,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f2f634c177d142b1963fe9f4ee3a8a70 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f2f634c177d142b1963fe9f4ee3a8a70 2024-11-13T05:51:45,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/bc83be3e014242ac93f0ab08dac6e060 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/bc83be3e014242ac93f0ab08dac6e060 2024-11-13T05:51:45,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/24ac1f56cda8444098b95877d6f8015c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/24ac1f56cda8444098b95877d6f8015c 2024-11-13T05:51:45,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/abb73ca8b9524211afcba33c092728fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/abb73ca8b9524211afcba33c092728fd 2024-11-13T05:51:45,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2dddada9bc5e4908ae3ffa1a49e08b53 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2dddada9bc5e4908ae3ffa1a49e08b53 2024-11-13T05:51:45,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/82781f0a4d6644708c5e0c6e413bbcb3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/82781f0a4d6644708c5e0c6e413bbcb3 2024-11-13T05:51:45,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/9db059423ede4a639e7c0b59ff6d54c9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/9db059423ede4a639e7c0b59ff6d54c9 2024-11-13T05:51:45,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b4b9cca6878b484bbfbfb61f7119a802 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/b4b9cca6878b484bbfbfb61f7119a802 2024-11-13T05:51:45,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/c73c23edd1ca46bea0e2156c5badb9f0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/c73c23edd1ca46bea0e2156c5badb9f0 2024-11-13T05:51:45,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/75ee8da83aeb4a9d868bb9dd254f4170 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/75ee8da83aeb4a9d868bb9dd254f4170 2024-11-13T05:51:45,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/73a4a986a4504f51a5e7561260eab6f5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/73a4a986a4504f51a5e7561260eab6f5 2024-11-13T05:51:45,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d7fac002150f42cd8f02bc5b52922cae to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d7fac002150f42cd8f02bc5b52922cae 2024-11-13T05:51:45,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f6cd53308342434eafceb86bda70f96a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/f6cd53308342434eafceb86bda70f96a 2024-11-13T05:51:45,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/4343648bbc6f4832a5b42b55e058d053 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/4343648bbc6f4832a5b42b55e058d053 2024-11-13T05:51:45,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/1c34657f11474080ba22ead3063d18be to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/1c34657f11474080ba22ead3063d18be 2024-11-13T05:51:45,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/989c8fc493aa4ff29e5bd58ac86de779 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/989c8fc493aa4ff29e5bd58ac86de779 2024-11-13T05:51:45,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d24b26bfcb9f49c884cd7fb78342539e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/d24b26bfcb9f49c884cd7fb78342539e 2024-11-13T05:51:45,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/edabbe98bbb145a6b93ccbe226a55fc6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/edabbe98bbb145a6b93ccbe226a55fc6 2024-11-13T05:51:45,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2122e81f6e084318bbcf36239757de3e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/2122e81f6e084318bbcf36239757de3e 2024-11-13T05:51:45,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f8c3637cbc7f4b8d9e51862d37b86c90, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4404b1b8f70b4e8c8e68d100abb55868, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/de9caee55791481d8d5e8f97227a355a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/ab3e5f31d8ef4e7092fc688b8ea0b315, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/90e14dfabcf54febaa1085ef6081e326, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/115520e507de48d5a63c544d645c270b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4b78d436c3e480fb21e889b57088179, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/715fb1f81c004d94982b4839a41b6bb2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b3912b0b9f2d4936b93e8172acbfefcc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/6efb87d8b60f4088b7818007f4a7f6a0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd5f3eb9b1b84e8b89b2323016cc771c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/77d90afdc0524db7936129eee01f4900, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/a0cda591888d4a4fa0dc08ad90339b0a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d0673a13857b4b0ba7817afb71cd155b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/e8b856a70e224ad0901c3b314e195d65, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/516b8c6e5c704443929bc40486c4545e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4ee5bc71bc842409af24e4d07556665, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/3fc080db79c0423b830c6a8e3193742d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9d9f39fd78ad4b3b8b6474144968ac27, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/28630f86867c4dc5a8419b5b359a3f42, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/893cd25b7085456ba107f1670efcce06, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd0ee2e7b52a463cb9d4094d0b6099dd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/c76cd6bf571b43dc83bb8c321eaa8cde, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/414ce880f19d4435a89beb94ddc28ade] to archive 2024-11-13T05:51:45,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:51:45,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f8c3637cbc7f4b8d9e51862d37b86c90 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f8c3637cbc7f4b8d9e51862d37b86c90 2024-11-13T05:51:45,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4404b1b8f70b4e8c8e68d100abb55868 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4404b1b8f70b4e8c8e68d100abb55868 2024-11-13T05:51:45,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/de9caee55791481d8d5e8f97227a355a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/de9caee55791481d8d5e8f97227a355a 2024-11-13T05:51:45,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/ab3e5f31d8ef4e7092fc688b8ea0b315 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/ab3e5f31d8ef4e7092fc688b8ea0b315 2024-11-13T05:51:45,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/90e14dfabcf54febaa1085ef6081e326 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/90e14dfabcf54febaa1085ef6081e326 2024-11-13T05:51:45,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/115520e507de48d5a63c544d645c270b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/115520e507de48d5a63c544d645c270b 2024-11-13T05:51:45,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9bccaa3a0dcf4629abaac0d1c8f0fb7e 2024-11-13T05:51:45,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4b78d436c3e480fb21e889b57088179 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4b78d436c3e480fb21e889b57088179 2024-11-13T05:51:45,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/715fb1f81c004d94982b4839a41b6bb2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/715fb1f81c004d94982b4839a41b6bb2 2024-11-13T05:51:45,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b3912b0b9f2d4936b93e8172acbfefcc to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b3912b0b9f2d4936b93e8172acbfefcc 2024-11-13T05:51:45,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/6efb87d8b60f4088b7818007f4a7f6a0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/6efb87d8b60f4088b7818007f4a7f6a0 2024-11-13T05:51:45,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd5f3eb9b1b84e8b89b2323016cc771c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd5f3eb9b1b84e8b89b2323016cc771c 2024-11-13T05:51:45,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/77d90afdc0524db7936129eee01f4900 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/77d90afdc0524db7936129eee01f4900 2024-11-13T05:51:45,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/a0cda591888d4a4fa0dc08ad90339b0a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/a0cda591888d4a4fa0dc08ad90339b0a 2024-11-13T05:51:45,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d0673a13857b4b0ba7817afb71cd155b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d0673a13857b4b0ba7817afb71cd155b 2024-11-13T05:51:45,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/e8b856a70e224ad0901c3b314e195d65 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/e8b856a70e224ad0901c3b314e195d65 2024-11-13T05:51:45,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/516b8c6e5c704443929bc40486c4545e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/516b8c6e5c704443929bc40486c4545e 2024-11-13T05:51:45,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4ee5bc71bc842409af24e4d07556665 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/b4ee5bc71bc842409af24e4d07556665 2024-11-13T05:51:45,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/3fc080db79c0423b830c6a8e3193742d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/3fc080db79c0423b830c6a8e3193742d 2024-11-13T05:51:45,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/12bfd2ac1c0f42b5a00f48ae1a8dd7a1 2024-11-13T05:51:45,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9d9f39fd78ad4b3b8b6474144968ac27 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/9d9f39fd78ad4b3b8b6474144968ac27 2024-11-13T05:51:45,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/28630f86867c4dc5a8419b5b359a3f42 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/28630f86867c4dc5a8419b5b359a3f42 2024-11-13T05:51:45,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/893cd25b7085456ba107f1670efcce06 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/893cd25b7085456ba107f1670efcce06 2024-11-13T05:51:45,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd0ee2e7b52a463cb9d4094d0b6099dd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/dd0ee2e7b52a463cb9d4094d0b6099dd 2024-11-13T05:51:45,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/c76cd6bf571b43dc83bb8c321eaa8cde to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/c76cd6bf571b43dc83bb8c321eaa8cde 2024-11-13T05:51:45,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/414ce880f19d4435a89beb94ddc28ade to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/414ce880f19d4435a89beb94ddc28ade 2024-11-13T05:51:45,395 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/recovered.edits/429.seqid, newMaxSeqId=429, maxSeqId=1 2024-11-13T05:51:45,395 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. 2024-11-13T05:51:45,395 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for f3802ebb1be649f045361204f3bf7507: Waiting for close lock at 1731477104012Running coprocessor pre-close hooks at 1731477104012Disabling compacts and flushes for region at 1731477104012Disabling writes for close at 1731477104013 (+1 ms)Obtaining lock to block concurrent updates at 1731477104013Preparing flush snapshotting stores in f3802ebb1be649f045361204f3bf7507 at 1731477104013Finished memstore snapshotting TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731477104014 (+1 ms)Flushing stores of TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507. at 1731477104015 (+1 ms)Flushing f3802ebb1be649f045361204f3bf7507/A: creating writer at 1731477104015Flushing f3802ebb1be649f045361204f3bf7507/A: appending metadata at 1731477104019 (+4 ms)Flushing f3802ebb1be649f045361204f3bf7507/A: closing flushed file at 1731477104019Flushing f3802ebb1be649f045361204f3bf7507/B: creating writer at 1731477104435 (+416 ms)Flushing f3802ebb1be649f045361204f3bf7507/B: appending metadata at 1731477104437 (+2 ms)Flushing f3802ebb1be649f045361204f3bf7507/B: closing flushed file at 1731477104437Flushing f3802ebb1be649f045361204f3bf7507/C: creating writer at 1731477104844 (+407 ms)Flushing f3802ebb1be649f045361204f3bf7507/C: appending metadata at 1731477104846 (+2 ms)Flushing f3802ebb1be649f045361204f3bf7507/C: closing flushed file at 1731477104846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eb3a58e: reopening flushed file at 1731477105288 (+442 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@449f50b: reopening flushed file at 1731477105295 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ed155c4: reopening flushed file at 1731477105301 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f3802ebb1be649f045361204f3bf7507 in 1295ms, sequenceid=426, compaction requested=true at 1731477105308 (+7 ms)Writing region close event to WAL at 1731477105392 (+84 ms)Running coprocessor post-close hooks at 1731477105395 (+3 ms)Closed at 1731477105395 2024-11-13T05:51:45,397 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:45,397 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f3802ebb1be649f045361204f3bf7507, regionState=CLOSED 2024-11-13T05:51:45,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:51:45,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T05:51:45,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure f3802ebb1be649f045361204f3bf7507, server=bc386b7bf2fe,38979,1731477016232 in 1.5390 sec 2024-11-13T05:51:45,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T05:51:45,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3802ebb1be649f045361204f3bf7507, UNASSIGN in 1.5430 sec 2024-11-13T05:51:45,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-13T05:51:45,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5460 sec 2024-11-13T05:51:45,404 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477105404"}]},"ts":"1731477105404"} 2024-11-13T05:51:45,405 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:51:45,405 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:51:45,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5600 sec 2024-11-13T05:51:45,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T05:51:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T05:51:46,002 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:46,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:51:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,006 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-13T05:51:46,007 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,010 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:46,015 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/recovered.edits] 2024-11-13T05:51:46,022 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/61d86206886e46a7b134a44fbbd6360c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/61d86206886e46a7b134a44fbbd6360c 2024-11-13T05:51:46,024 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c98911f9149d48e69734f5186e62f9e6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/c98911f9149d48e69734f5186e62f9e6 2024-11-13T05:51:46,026 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f15514f5e1264e2b81ced599fe67d511 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/A/f15514f5e1264e2b81ced599fe67d511 2024-11-13T05:51:46,029 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/56a6ade9022a43679a3ecaea9af1955b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/56a6ade9022a43679a3ecaea9af1955b 2024-11-13T05:51:46,030 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/e4322e59617a43b18496ac7ed91a32f3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/e4322e59617a43b18496ac7ed91a32f3 2024-11-13T05:51:46,031 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ff1a961e548346c896a67685eb3f125f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/B/ff1a961e548346c896a67685eb3f125f 2024-11-13T05:51:46,034 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4b451a083fc64d4f97fc2009e603ab1c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/4b451a083fc64d4f97fc2009e603ab1c 2024-11-13T05:51:46,035 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d02818bcec61408abd8c45e1ab6ece04 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/d02818bcec61408abd8c45e1ab6ece04 2024-11-13T05:51:46,036 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f59739f1230046a1b0b02c194b7e885a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/C/f59739f1230046a1b0b02c194b7e885a 2024-11-13T05:51:46,039 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/recovered.edits/429.seqid to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507/recovered.edits/429.seqid 2024-11-13T05:51:46,039 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/f3802ebb1be649f045361204f3bf7507 2024-11-13T05:51:46,040 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:51:46,041 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,044 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:51:46,046 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:51:46,047 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,047 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:51:46,047 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731477106047"}]},"ts":"9223372036854775807"} 2024-11-13T05:51:46,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:51:46,049 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f3802ebb1be649f045361204f3bf7507, NAME => 'TestAcidGuarantees,,1731477078260.f3802ebb1be649f045361204f3bf7507.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:51:46,049 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:51:46,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731477106049"}]},"ts":"9223372036854775807"} 2024-11-13T05:51:46,051 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:51:46,052 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-13T05:51:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-13T05:51:46,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:51:46,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:46,123 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 251), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 475), ProcessCount=11 (was 11), AvailableMemoryMB=6892 (was 7190) 2024-11-13T05:51:46,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=6891 2024-11-13T05:51:46,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:51:46,137 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:51:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:46,139 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:51:46,139 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:46,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-13T05:51:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:51:46,140 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:51:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742163_1339 (size=1061) 2024-11-13T05:51:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:51:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:51:46,552 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:51:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742164_1340 (size=53) 2024-11-13T05:51:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:51:46,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:51:46,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 49578a99aa8fa6a60ec17a19ef73c0fd, disabling compactions & flushes 2024-11-13T05:51:46,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:46,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:46,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. after waiting 0 ms 2024-11-13T05:51:46,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:46,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:46,964 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49578a99aa8fa6a60ec17a19ef73c0fd: Waiting for close lock at 1731477106964Disabling compacts and flushes for region at 1731477106964Disabling writes for close at 1731477106964Writing region close event to WAL at 1731477106964Closed at 1731477106964 2024-11-13T05:51:46,966 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:51:46,967 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731477106967"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477106967"}]},"ts":"1731477106967"} 2024-11-13T05:51:46,971 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:51:46,973 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:51:46,973 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477106973"}]},"ts":"1731477106973"} 2024-11-13T05:51:46,975 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:51:46,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, ASSIGN}] 2024-11-13T05:51:46,977 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, ASSIGN 2024-11-13T05:51:46,978 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=false 2024-11-13T05:51:47,129 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:47,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, ASSIGN because future has completed 2024-11-13T05:51:47,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:51:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:51:47,291 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:47,291 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:51:47,292 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,292 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:51:47,292 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,292 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,295 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,297 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:47,297 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49578a99aa8fa6a60ec17a19ef73c0fd columnFamilyName A 2024-11-13T05:51:47,298 DEBUG [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:47,298 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(327): Store=49578a99aa8fa6a60ec17a19ef73c0fd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:47,298 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,299 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:47,300 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49578a99aa8fa6a60ec17a19ef73c0fd columnFamilyName B 2024-11-13T05:51:47,300 DEBUG [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:47,300 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(327): Store=49578a99aa8fa6a60ec17a19ef73c0fd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:47,300 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,301 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:47,301 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49578a99aa8fa6a60ec17a19ef73c0fd columnFamilyName C 2024-11-13T05:51:47,301 DEBUG [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:47,302 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(327): Store=49578a99aa8fa6a60ec17a19ef73c0fd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:47,302 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:47,302 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,302 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,303 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,304 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,304 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,304 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:51:47,305 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,307 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:51:47,307 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 49578a99aa8fa6a60ec17a19ef73c0fd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60372916, jitterRate=-0.10037344694137573}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:51:47,307 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:47,308 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 49578a99aa8fa6a60ec17a19ef73c0fd: Running coprocessor pre-open hook at 1731477107292Writing region info on filesystem at 1731477107292Initializing all the Stores at 1731477107294 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477107294Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477107294Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477107295 (+1 ms)Cleaning up temporary data from old regions at 1731477107304 (+9 ms)Running coprocessor post-open hooks at 1731477107307 (+3 ms)Region opened successfully at 1731477107308 (+1 ms) 2024-11-13T05:51:47,309 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., pid=93, masterSystemTime=1731477107287 2024-11-13T05:51:47,310 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:47,310 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:47,311 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:47,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:51:47,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-13T05:51:47,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 in 180 msec 2024-11-13T05:51:47,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T05:51:47,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, ASSIGN in 339 msec 2024-11-13T05:51:47,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:51:47,316 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477107316"}]},"ts":"1731477107316"} 2024-11-13T05:51:47,317 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:51:47,318 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:51:47,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-13T05:51:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-13T05:51:48,292 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:48,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:48,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:48,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:48,295 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb55bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:48,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f35c11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:48,299 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:48,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:48,303 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:51:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:51:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@777809f7 2024-11-13T05:51:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:51:48,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:51:48,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:51:48,306 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:51:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T05:51:48,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742165_1341 (size=1097) 2024-11-13T05:51:48,720 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T05:51:48,720 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T05:51:48,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:51:48,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, REOPEN/MOVE}] 2024-11-13T05:51:48,730 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, REOPEN/MOVE 2024-11-13T05:51:48,731 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:48,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, REOPEN/MOVE because future has completed 2024-11-13T05:51:48,734 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:51:48,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:51:48,887 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:48,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:51:48,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 49578a99aa8fa6a60ec17a19ef73c0fd, disabling compactions & flushes 2024-11-13T05:51:48,887 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:48,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:48,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. after waiting 0 ms 2024-11-13T05:51:48,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:48,891 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:51:48,891 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:48,891 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 49578a99aa8fa6a60ec17a19ef73c0fd: Waiting for close lock at 1731477108887Running coprocessor pre-close hooks at 1731477108887Disabling compacts and flushes for region at 1731477108887Disabling writes for close at 1731477108887Writing region close event to WAL at 1731477108887Running coprocessor post-close hooks at 1731477108891 (+4 ms)Closed at 1731477108891 2024-11-13T05:51:48,892 WARN [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 49578a99aa8fa6a60ec17a19ef73c0fd to self. 2024-11-13T05:51:48,894 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:48,894 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=CLOSED 2024-11-13T05:51:48,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:51:48,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-13T05:51:48,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 in 163 msec 2024-11-13T05:51:48,900 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=true 2024-11-13T05:51:49,051 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, REOPEN/MOVE because future has completed 2024-11-13T05:51:49,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:51:49,213 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,213 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:51:49,213 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,213 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:51:49,213 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,213 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,214 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,215 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:49,215 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49578a99aa8fa6a60ec17a19ef73c0fd columnFamilyName A 2024-11-13T05:51:49,216 DEBUG [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:49,216 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(327): Store=49578a99aa8fa6a60ec17a19ef73c0fd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:49,217 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,217 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:49,217 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49578a99aa8fa6a60ec17a19ef73c0fd columnFamilyName B 2024-11-13T05:51:49,217 DEBUG [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:49,218 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(327): Store=49578a99aa8fa6a60ec17a19ef73c0fd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:49,218 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,218 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:51:49,218 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49578a99aa8fa6a60ec17a19ef73c0fd columnFamilyName C 2024-11-13T05:51:49,218 DEBUG [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:49,219 INFO [StoreOpener-49578a99aa8fa6a60ec17a19ef73c0fd-1 {}] regionserver.HStore(327): Store=49578a99aa8fa6a60ec17a19ef73c0fd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:51:49,219 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,219 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,219 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,220 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,221 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,221 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,222 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:51:49,223 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,224 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 49578a99aa8fa6a60ec17a19ef73c0fd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62663339, jitterRate=-0.0662434846162796}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:51:49,224 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,224 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 49578a99aa8fa6a60ec17a19ef73c0fd: Running coprocessor pre-open hook at 1731477109213Writing region info on filesystem at 1731477109213Initializing all the Stores at 1731477109214 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477109214Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477109214Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477109214Cleaning up temporary data from old regions at 1731477109221 (+7 ms)Running coprocessor post-open hooks at 1731477109224 (+3 ms)Region opened successfully at 1731477109224 2024-11-13T05:51:49,225 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., pid=98, masterSystemTime=1731477109210 2024-11-13T05:51:49,227 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,227 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,227 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:51:49,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-13T05:51:49,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 in 173 msec 2024-11-13T05:51:49,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T05:51:49,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, REOPEN/MOVE in 502 msec 2024-11-13T05:51:49,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-13T05:51:49,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-13T05:51:49,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-13T05:51:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-13T05:51:49,237 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:49,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,239 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2af295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3581f00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f147575, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eceeeff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,245 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91edd69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc083d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b45c8be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,248 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e3c373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4dd19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735e65c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,252 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1255d9d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,252 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1ee25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,254 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3cb922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577e421c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e1d45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4ed5ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41439d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690cddc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75840ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f3f328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@845334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:51:49,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:51:49,265 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:51:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:51:49,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:51:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e2728f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:51:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:51:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:51:49,267 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:51:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fbd21c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:51:49,268 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,268 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:49,268 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,269 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,269 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,269 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T05:51:49,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,270 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,270 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:49,271 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:49,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,272 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:49,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,273 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,273 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:51:49,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:51:49,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:51:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:51:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:51:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T05:51:49,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:49,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:49,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:49,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:49,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:49,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:49,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111326c44876bc764a0fb0333a4b5c80b853_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477109283/Put/seqid=0 2024-11-13T05:51:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477169313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742166_1342 (size=21918) 2024-11-13T05:51:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477169316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477169316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477169317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477169322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:49,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:49,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:49,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477169431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477169431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477169431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477169431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477169432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:49,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477169642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477169642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477169642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477169642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477169642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,723 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:49,726 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111326c44876bc764a0fb0333a4b5c80b853_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111326c44876bc764a0fb0333a4b5c80b853_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:49,727 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/fc6bc1fea53f4dba95eaa603fb72004f, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:49,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/fc6bc1fea53f4dba95eaa603fb72004f is 175, key is test_row_0/A:col10/1731477109283/Put/seqid=0 2024-11-13T05:51:49,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742167_1343 (size=65323) 2024-11-13T05:51:49,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/fc6bc1fea53f4dba95eaa603fb72004f 2024-11-13T05:51:49,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:49,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/6e41847900464fb3b4e6a32718b965eb is 50, key is test_row_0/B:col10/1731477109283/Put/seqid=0 2024-11-13T05:51:49,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742168_1344 (size=12001) 2024-11-13T05:51:49,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:49,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:49,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477169951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477169951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477169951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477169952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477169952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:50,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/6e41847900464fb3b4e6a32718b965eb 2024-11-13T05:51:50,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/2d21f2fbc05441e881f7da326f42d18b is 50, key is test_row_0/C:col10/1731477109283/Put/seqid=0 2024-11-13T05:51:50,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742169_1345 (size=12001) 2024-11-13T05:51:50,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:50,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477170462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477170462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477170462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477170462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477170462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:50,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:50,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/2d21f2fbc05441e881f7da326f42d18b 2024-11-13T05:51:50,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/fc6bc1fea53f4dba95eaa603fb72004f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f 2024-11-13T05:51:50,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f, entries=350, sequenceid=16, filesize=63.8 K 2024-11-13T05:51:50,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/6e41847900464fb3b4e6a32718b965eb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6e41847900464fb3b4e6a32718b965eb 2024-11-13T05:51:50,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6e41847900464fb3b4e6a32718b965eb, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:51:50,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/2d21f2fbc05441e881f7da326f42d18b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2d21f2fbc05441e881f7da326f42d18b 2024-11-13T05:51:50,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2d21f2fbc05441e881f7da326f42d18b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T05:51:50,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1310ms, sequenceid=16, compaction requested=false 2024-11-13T05:51:50,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:50,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T05:51:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:50,658 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:51:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:50,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132173226fed704edf95d993e3eec2b60a_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477109315/Put/seqid=0 2024-11-13T05:51:50,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742170_1346 (size=12154) 2024-11-13T05:51:50,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:50,684 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132173226fed704edf95d993e3eec2b60a_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132173226fed704edf95d993e3eec2b60a_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/06d925024a2749a391a0c6f5e5fb2771, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:50,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/06d925024a2749a391a0c6f5e5fb2771 is 175, key is test_row_0/A:col10/1731477109315/Put/seqid=0 2024-11-13T05:51:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742171_1347 (size=30955) 2024-11-13T05:51:50,701 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/06d925024a2749a391a0c6f5e5fb2771 2024-11-13T05:51:50,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/52063ecdfaf345b09333fe8a18927755 is 50, key is test_row_0/B:col10/1731477109315/Put/seqid=0 2024-11-13T05:51:50,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742172_1348 (size=12001) 2024-11-13T05:51:50,921 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:51:51,118 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/52063ecdfaf345b09333fe8a18927755 2024-11-13T05:51:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/6e5bf9889c9644fa99e1e58400868b50 is 50, key is test_row_0/C:col10/1731477109315/Put/seqid=0 2024-11-13T05:51:51,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742173_1349 (size=12001) 2024-11-13T05:51:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:51,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:51,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477171487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477171488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477171489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477171490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477171490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,574 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/6e5bf9889c9644fa99e1e58400868b50 2024-11-13T05:51:51,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/06d925024a2749a391a0c6f5e5fb2771 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771 2024-11-13T05:51:51,584 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771, entries=150, sequenceid=36, filesize=30.2 K 2024-11-13T05:51:51,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/52063ecdfaf345b09333fe8a18927755 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/52063ecdfaf345b09333fe8a18927755 2024-11-13T05:51:51,588 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/52063ecdfaf345b09333fe8a18927755, entries=150, sequenceid=36, filesize=11.7 K 2024-11-13T05:51:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/6e5bf9889c9644fa99e1e58400868b50 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/6e5bf9889c9644fa99e1e58400868b50 2024-11-13T05:51:51,595 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/6e5bf9889c9644fa99e1e58400868b50, entries=150, sequenceid=36, filesize=11.7 K 2024-11-13T05:51:51,597 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 49578a99aa8fa6a60ec17a19ef73c0fd in 939ms, sequenceid=36, compaction requested=false 2024-11-13T05:51:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T05:51:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T05:51:51,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T05:51:51,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3270 sec 2024-11-13T05:51:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:51,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:51:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:51,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:51,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3350 sec 2024-11-13T05:51:51,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133f6d6bf1d0f34dff8e2b908ea9264760_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742174_1350 (size=14594) 2024-11-13T05:51:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477171637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477171638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477171639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477171649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477171649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477171751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477171761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477171761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477171762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477171762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477171971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477171971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477171971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477171972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477171972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:52,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133f6d6bf1d0f34dff8e2b908ea9264760_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f6d6bf1d0f34dff8e2b908ea9264760_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:52,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0b35b7a6198b4bec939437979723a681, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:52,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0b35b7a6198b4bec939437979723a681 is 175, key is test_row_0/A:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:52,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742175_1351 (size=39549) 2024-11-13T05:51:52,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0b35b7a6198b4bec939437979723a681 2024-11-13T05:51:52,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/4815e033edda4d1ca446e4578bdc731a is 50, key is test_row_0/B:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:52,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742176_1352 (size=12001) 2024-11-13T05:51:52,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477172281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477172281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477172281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477172281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477172282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/4815e033edda4d1ca446e4578bdc731a 2024-11-13T05:51:52,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/e6f9a22018c54036862b95ef2a902988 is 50, key is test_row_0/C:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:52,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742177_1353 (size=12001) 2024-11-13T05:51:52,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477172791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477172791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477172802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477172802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477172802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:52,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/e6f9a22018c54036862b95ef2a902988 2024-11-13T05:51:52,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0b35b7a6198b4bec939437979723a681 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681 2024-11-13T05:51:52,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681, entries=200, sequenceid=49, filesize=38.6 K 2024-11-13T05:51:52,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/4815e033edda4d1ca446e4578bdc731a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4815e033edda4d1ca446e4578bdc731a 2024-11-13T05:51:52,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4815e033edda4d1ca446e4578bdc731a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T05:51:52,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/e6f9a22018c54036862b95ef2a902988 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e6f9a22018c54036862b95ef2a902988 2024-11-13T05:51:52,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e6f9a22018c54036862b95ef2a902988, entries=150, sequenceid=49, filesize=11.7 K 2024-11-13T05:51:52,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1266ms, sequenceid=49, compaction requested=true 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:52,870 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:52,870 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:52,871 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:52,871 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:51:52,871 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:52,871 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=132.6 K 2024-11-13T05:51:52,871 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:52,871 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:52,871 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:51:52,871 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:52,871 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681] 2024-11-13T05:51:52,871 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6e41847900464fb3b4e6a32718b965eb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/52063ecdfaf345b09333fe8a18927755, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4815e033edda4d1ca446e4578bdc731a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=35.2 K 2024-11-13T05:51:52,872 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc6bc1fea53f4dba95eaa603fb72004f, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731477109277 2024-11-13T05:51:52,872 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e41847900464fb3b4e6a32718b965eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731477109283 2024-11-13T05:51:52,872 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d925024a2749a391a0c6f5e5fb2771, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731477109301 2024-11-13T05:51:52,872 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 52063ecdfaf345b09333fe8a18927755, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731477109301 2024-11-13T05:51:52,872 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b35b7a6198b4bec939437979723a681, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477111485 2024-11-13T05:51:52,872 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4815e033edda4d1ca446e4578bdc731a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477111488 2024-11-13T05:51:52,876 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:52,878 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:52,879 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/3e073246f37f446db6c918addb106f28 is 50, key is test_row_0/B:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:52,881 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c2a614088b3d461783765dec2c9c4c1d_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:52,883 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c2a614088b3d461783765dec2c9c4c1d_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:52,884 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2a614088b3d461783765dec2c9c4c1d_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:52,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742178_1354 (size=12104) 2024-11-13T05:51:52,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742179_1355 (size=4469) 2024-11-13T05:51:53,288 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:53,289 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0a9a99700fc7486b9ad675ecbd4e18bc is 175, key is test_row_0/A:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:53,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742180_1356 (size=31058) 2024-11-13T05:51:53,293 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/3e073246f37f446db6c918addb106f28 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3e073246f37f446db6c918addb106f28 2024-11-13T05:51:53,297 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into 3e073246f37f446db6c918addb106f28(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:53,297 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:53,297 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477112870; duration=0sec 2024-11-13T05:51:53,297 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:53,297 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:51:53,297 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:53,299 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:53,299 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:51:53,299 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:53,300 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2d21f2fbc05441e881f7da326f42d18b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/6e5bf9889c9644fa99e1e58400868b50, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e6f9a22018c54036862b95ef2a902988] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=35.2 K 2024-11-13T05:51:53,300 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d21f2fbc05441e881f7da326f42d18b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731477109283 2024-11-13T05:51:53,300 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e5bf9889c9644fa99e1e58400868b50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731477109301 2024-11-13T05:51:53,301 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f9a22018c54036862b95ef2a902988, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477111488 2024-11-13T05:51:53,314 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:53,314 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/99ac8ab98b8342efb3a80a74522d017e is 50, key is test_row_0/C:col10/1731477111489/Put/seqid=0 2024-11-13T05:51:53,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742181_1357 (size=12104) 2024-11-13T05:51:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T05:51:53,432 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:53,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T05:51:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T05:51:53,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:53,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:53,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T05:51:53,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T05:51:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:53,587 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:51:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:53,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111309747f91e37a4951b0979fbe948b02ef_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477111648/Put/seqid=0 2024-11-13T05:51:53,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742182_1358 (size=12154) 2024-11-13T05:51:53,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:53,609 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111309747f91e37a4951b0979fbe948b02ef_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111309747f91e37a4951b0979fbe948b02ef_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0e1c45197653426689035fc7d7e0dbd8, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0e1c45197653426689035fc7d7e0dbd8 is 175, key is test_row_0/A:col10/1731477111648/Put/seqid=0 2024-11-13T05:51:53,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742183_1359 (size=30955) 2024-11-13T05:51:53,619 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0e1c45197653426689035fc7d7e0dbd8 2024-11-13T05:51:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/71fbca0ac9434ff1b104e70a7210b775 is 50, key is test_row_0/B:col10/1731477111648/Put/seqid=0 2024-11-13T05:51:53,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742184_1360 (size=12001) 2024-11-13T05:51:53,697 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0a9a99700fc7486b9ad675ecbd4e18bc as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0a9a99700fc7486b9ad675ecbd4e18bc 2024-11-13T05:51:53,701 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into 0a9a99700fc7486b9ad675ecbd4e18bc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:53,701 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:53,701 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477112870; duration=0sec 2024-11-13T05:51:53,701 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:53,701 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:51:53,721 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/99ac8ab98b8342efb3a80a74522d017e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/99ac8ab98b8342efb3a80a74522d017e 2024-11-13T05:51:53,725 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into 99ac8ab98b8342efb3a80a74522d017e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:53,725 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:53,725 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477112870; duration=0sec 2024-11-13T05:51:53,725 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:53,725 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:51:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T05:51:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:53,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:53,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477173821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477173823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,832 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477173831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477173832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477173832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477173942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477173942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477173942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:53,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477173951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,032 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/71fbca0ac9434ff1b104e70a7210b775 2024-11-13T05:51:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986 is 50, key is test_row_0/C:col10/1731477111648/Put/seqid=0 2024-11-13T05:51:54,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742185_1361 (size=12001) 2024-11-13T05:51:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T05:51:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477174151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477174161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477174161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477174171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,443 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986 2024-11-13T05:51:54,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0e1c45197653426689035fc7d7e0dbd8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8 2024-11-13T05:51:54,450 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8, entries=150, sequenceid=70, filesize=30.2 K 2024-11-13T05:51:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/71fbca0ac9434ff1b104e70a7210b775 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/71fbca0ac9434ff1b104e70a7210b775 2024-11-13T05:51:54,454 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/71fbca0ac9434ff1b104e70a7210b775, entries=150, sequenceid=70, filesize=11.7 K 2024-11-13T05:51:54,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986 2024-11-13T05:51:54,460 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986, entries=150, sequenceid=70, filesize=11.7 K 2024-11-13T05:51:54,461 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 49578a99aa8fa6a60ec17a19ef73c0fd in 875ms, sequenceid=70, compaction requested=false 2024-11-13T05:51:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T05:51:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T05:51:54,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T05:51:54,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-11-13T05:51:54,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.0320 sec 2024-11-13T05:51:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:54,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:51:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:54,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1bd823c7a524b3186c97c26e0570f96_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:54,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742186_1362 (size=14594) 2024-11-13T05:51:54,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:54,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b1bd823c7a524b3186c97c26e0570f96_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1bd823c7a524b3186c97c26e0570f96_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:54,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87175c1e26fb4973a866b787ca72e140, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:54,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87175c1e26fb4973a866b787ca72e140 is 175, key is test_row_0/A:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:54,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742187_1363 (size=39549) 2024-11-13T05:51:54,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477174518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477174518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477174522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477174530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T05:51:54,571 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:54,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T05:51:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:51:54,573 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:54,573 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:54,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477174641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477174641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477174641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477174641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:51:54,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:54,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477174861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477174861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477174861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477174861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:54,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:54,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:51:54,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87175c1e26fb4973a866b787ca72e140 2024-11-13T05:51:54,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/d99a255b70e24e2e8315faaa76db1d11 is 50, key is test_row_0/B:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:54,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742188_1364 (size=12001) 2024-11-13T05:51:55,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:55,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477175172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477175172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477175172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477175172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:51:55,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/d99a255b70e24e2e8315faaa76db1d11 2024-11-13T05:51:55,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a08f121a86104282a8342eeefb0c1bbb is 50, key is test_row_0/C:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742189_1365 (size=12001) 2024-11-13T05:51:55,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:55,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477175691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477175691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477175691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477175691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:51:55,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a08f121a86104282a8342eeefb0c1bbb 2024-11-13T05:51:55,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87175c1e26fb4973a866b787ca72e140 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140 2024-11-13T05:51:55,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140, entries=200, sequenceid=86, filesize=38.6 K 2024-11-13T05:51:55,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/d99a255b70e24e2e8315faaa76db1d11 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d99a255b70e24e2e8315faaa76db1d11 2024-11-13T05:51:55,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d99a255b70e24e2e8315faaa76db1d11, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T05:51:55,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a08f121a86104282a8342eeefb0c1bbb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a08f121a86104282a8342eeefb0c1bbb 2024-11-13T05:51:55,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a08f121a86104282a8342eeefb0c1bbb, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T05:51:55,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1271ms, sequenceid=86, compaction requested=true 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:55,747 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:55,747 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:55,748 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:55,748 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:55,748 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:51:55,748 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:51:55,748 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,748 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,749 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3e073246f37f446db6c918addb106f28, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/71fbca0ac9434ff1b104e70a7210b775, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d99a255b70e24e2e8315faaa76db1d11] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=35.3 K 2024-11-13T05:51:55,749 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0a9a99700fc7486b9ad675ecbd4e18bc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=99.2 K 2024-11-13T05:51:55,749 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0a9a99700fc7486b9ad675ecbd4e18bc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140] 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e073246f37f446db6c918addb106f28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477111488 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9a99700fc7486b9ad675ecbd4e18bc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477111488 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 71fbca0ac9434ff1b104e70a7210b775, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731477111637 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1c45197653426689035fc7d7e0dbd8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731477111637 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d99a255b70e24e2e8315faaa76db1d11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731477113816 2024-11-13T05:51:55,749 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87175c1e26fb4973a866b787ca72e140, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731477113816 2024-11-13T05:51:55,754 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:55,755 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:55,756 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/5ea7514035ac40aa860fae73c6742941 is 50, key is test_row_0/B:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:55,762 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113f850d367e56e4dffa0bf9fbd7b840c1c_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:55,764 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113f850d367e56e4dffa0bf9fbd7b840c1c_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:55,764 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f850d367e56e4dffa0bf9fbd7b840c1c_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:55,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742190_1366 (size=12207) 2024-11-13T05:51:55,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742191_1367 (size=4469) 2024-11-13T05:51:55,775 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#310 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:55,776 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/a41a97e1cf6a4be1a89ac6720c6dc587 is 175, key is test_row_0/A:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:55,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742192_1368 (size=31161) 2024-11-13T05:51:55,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T05:51:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:55,794 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T05:51:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:55,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111376761befd9114d3496903cd16d0fc383_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477114529/Put/seqid=0 2024-11-13T05:51:55,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742193_1369 (size=12154) 2024-11-13T05:51:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:55,807 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111376761befd9114d3496903cd16d0fc383_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376761befd9114d3496903cd16d0fc383_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/4e7c17905e1e4fd4800456b6cfcb9da2, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:55,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/4e7c17905e1e4fd4800456b6cfcb9da2 is 175, key is test_row_0/A:col10/1731477114529/Put/seqid=0 2024-11-13T05:51:55,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T05:51:55,814 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T05:51:55,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742194_1370 (size=30955) 2024-11-13T05:51:55,820 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/4e7c17905e1e4fd4800456b6cfcb9da2 2024-11-13T05:51:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/454e00e9bb8a4aa6b32d3772f4081fe6 is 50, key is test_row_0/B:col10/1731477114529/Put/seqid=0 2024-11-13T05:51:55,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742195_1371 (size=12001) 2024-11-13T05:51:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:55,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:55,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477175918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477176042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,169 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/5ea7514035ac40aa860fae73c6742941 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/5ea7514035ac40aa860fae73c6742941 2024-11-13T05:51:56,174 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into 5ea7514035ac40aa860fae73c6742941(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:56,174 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:56,174 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477115747; duration=0sec 2024-11-13T05:51:56,174 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:56,174 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:51:56,174 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:56,175 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:56,175 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:51:56,175 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:56,175 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/99ac8ab98b8342efb3a80a74522d017e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a08f121a86104282a8342eeefb0c1bbb] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=35.3 K 2024-11-13T05:51:56,175 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ac8ab98b8342efb3a80a74522d017e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731477111488 2024-11-13T05:51:56,176 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a6eb7f0609ae4a45ad3e1f7f4bdc7986, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731477111637 2024-11-13T05:51:56,176 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a08f121a86104282a8342eeefb0c1bbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731477113816 2024-11-13T05:51:56,183 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:56,183 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/983a22407bbe4089a2a625ea2d6322ca is 50, key is test_row_0/C:col10/1731477114474/Put/seqid=0 2024-11-13T05:51:56,186 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/a41a97e1cf6a4be1a89ac6720c6dc587 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a41a97e1cf6a4be1a89ac6720c6dc587 2024-11-13T05:51:56,191 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into a41a97e1cf6a4be1a89ac6720c6dc587(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:56,191 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:56,191 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477115747; duration=0sec 2024-11-13T05:51:56,191 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:56,191 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:51:56,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742196_1372 (size=12207) 2024-11-13T05:51:56,206 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/983a22407bbe4089a2a625ea2d6322ca as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/983a22407bbe4089a2a625ea2d6322ca 2024-11-13T05:51:56,212 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into 983a22407bbe4089a2a625ea2d6322ca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:56,212 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:56,213 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477115747; duration=0sec 2024-11-13T05:51:56,213 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:56,213 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:51:56,236 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/454e00e9bb8a4aa6b32d3772f4081fe6 2024-11-13T05:51:56,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a97aa9dc8b234177baef7d16e2c9857c is 50, key is test_row_0/C:col10/1731477114529/Put/seqid=0 2024-11-13T05:51:56,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742197_1373 (size=12001) 2024-11-13T05:51:56,257 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a97aa9dc8b234177baef7d16e2c9857c 2024-11-13T05:51:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/4e7c17905e1e4fd4800456b6cfcb9da2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2 2024-11-13T05:51:56,269 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2, entries=150, sequenceid=104, filesize=30.2 K 2024-11-13T05:51:56,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477176261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/454e00e9bb8a4aa6b32d3772f4081fe6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/454e00e9bb8a4aa6b32d3772f4081fe6 2024-11-13T05:51:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,279 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/454e00e9bb8a4aa6b32d3772f4081fe6, entries=150, sequenceid=104, filesize=11.7 K 2024-11-13T05:51:56,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a97aa9dc8b234177baef7d16e2c9857c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a97aa9dc8b234177baef7d16e2c9857c 2024-11-13T05:51:56,286 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a97aa9dc8b234177baef7d16e2c9857c, entries=150, sequenceid=104, filesize=11.7 K 2024-11-13T05:51:56,287 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 49578a99aa8fa6a60ec17a19ef73c0fd in 493ms, sequenceid=104, compaction requested=false 2024-11-13T05:51:56,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:56,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:56,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T05:51:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T05:51:56,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T05:51:56,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-13T05:51:56,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.7180 sec 2024-11-13T05:51:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:56,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:51:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:56,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:56,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138a649ad530bd434cb59974526753a5c7_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:56,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742198_1374 (size=14594) 2024-11-13T05:51:56,598 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:56,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138a649ad530bd434cb59974526753a5c7_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138a649ad530bd434cb59974526753a5c7_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:56,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/10cf68fc038e4e5ea99dee5f34e4c359, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:56,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/10cf68fc038e4e5ea99dee5f34e4c359 is 175, key is test_row_0/A:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:56,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742199_1375 (size=39549) 2024-11-13T05:51:56,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477176689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477176702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,709 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477176711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477176711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477176711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,714 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,714 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,715 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T05:51:56,721 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:56,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T05:51:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T05:51:56,723 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:56,724 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:56,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:56,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477176801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T05:51:56,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T05:51:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:56,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:56,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,007 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/10cf68fc038e4e5ea99dee5f34e4c359 2024-11-13T05:51:57,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/366bf23b5e354485aefee5b253999e68 is 50, key is test_row_0/B:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:57,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742200_1376 (size=12001) 2024-11-13T05:51:57,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/366bf23b5e354485aefee5b253999e68 2024-11-13T05:51:57,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T05:51:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477177021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:57,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/5b448dec1cbb49f889169c39a0a71409 is 50, key is test_row_0/C:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:57,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742201_1377 (size=12001) 2024-11-13T05:51:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T05:51:57,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T05:51:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T05:51:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:51:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477177341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T05:51:57,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/5b448dec1cbb49f889169c39a0a71409 2024-11-13T05:51:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/10cf68fc038e4e5ea99dee5f34e4c359 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359 2024-11-13T05:51:57,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359, entries=200, sequenceid=122, filesize=38.6 K 2024-11-13T05:51:57,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/366bf23b5e354485aefee5b253999e68 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/366bf23b5e354485aefee5b253999e68 2024-11-13T05:51:57,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/366bf23b5e354485aefee5b253999e68, entries=150, sequenceid=122, filesize=11.7 K 2024-11-13T05:51:57,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/5b448dec1cbb49f889169c39a0a71409 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5b448dec1cbb49f889169c39a0a71409 2024-11-13T05:51:57,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5b448dec1cbb49f889169c39a0a71409, entries=150, sequenceid=122, filesize=11.7 K 2024-11-13T05:51:57,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 49578a99aa8fa6a60ec17a19ef73c0fd in 866ms, sequenceid=122, compaction requested=true 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:51:57,454 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:51:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:51:57,454 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:57,454 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:51:57,454 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,455 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a41a97e1cf6a4be1a89ac6720c6dc587, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=99.3 K 2024-11-13T05:51:57,455 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,455 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a41a97e1cf6a4be1a89ac6720c6dc587, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359] 2024-11-13T05:51:57,455 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a41a97e1cf6a4be1a89ac6720c6dc587, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731477113816 2024-11-13T05:51:57,455 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e7c17905e1e4fd4800456b6cfcb9da2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731477114515 2024-11-13T05:51:57,455 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10cf68fc038e4e5ea99dee5f34e4c359, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731477115884 2024-11-13T05:51:57,456 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:57,456 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:57,456 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:51:57,457 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,457 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/5ea7514035ac40aa860fae73c6742941, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/454e00e9bb8a4aa6b32d3772f4081fe6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/366bf23b5e354485aefee5b253999e68] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=35.4 K 2024-11-13T05:51:57,457 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ea7514035ac40aa860fae73c6742941, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731477113816 2024-11-13T05:51:57,457 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 454e00e9bb8a4aa6b32d3772f4081fe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731477114515 2024-11-13T05:51:57,457 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 366bf23b5e354485aefee5b253999e68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731477115884 2024-11-13T05:51:57,460 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:57,464 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411131e96dc3699fb422a9084deee0ecc19c5_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:57,466 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:57,466 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/cd6dd74614b04807af9061792c6be342 is 50, key is test_row_0/B:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:57,466 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411131e96dc3699fb422a9084deee0ecc19c5_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:57,466 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e96dc3699fb422a9084deee0ecc19c5_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:57,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742202_1378 (size=12309) 2024-11-13T05:51:57,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742203_1379 (size=4469) 2024-11-13T05:51:57,476 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#318 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:57,476 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/760f2dce4cc34ba892a38644d6cd17fa is 175, key is test_row_0/A:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742204_1380 (size=31263) 2024-11-13T05:51:57,487 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/cd6dd74614b04807af9061792c6be342 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/cd6dd74614b04807af9061792c6be342 2024-11-13T05:51:57,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T05:51:57,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,490 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:51:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:57,492 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into cd6dd74614b04807af9061792c6be342(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:57,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:57,492 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477117454; duration=0sec 2024-11-13T05:51:57,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:51:57,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:51:57,492 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:51:57,493 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/760f2dce4cc34ba892a38644d6cd17fa as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/760f2dce4cc34ba892a38644d6cd17fa 2024-11-13T05:51:57,493 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:51:57,493 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:51:57,493 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:57,493 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/983a22407bbe4089a2a625ea2d6322ca, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a97aa9dc8b234177baef7d16e2c9857c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5b448dec1cbb49f889169c39a0a71409] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=35.4 K 2024-11-13T05:51:57,494 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 983a22407bbe4089a2a625ea2d6322ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731477113816 2024-11-13T05:51:57,494 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a97aa9dc8b234177baef7d16e2c9857c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731477114515 2024-11-13T05:51:57,495 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b448dec1cbb49f889169c39a0a71409, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731477115884 2024-11-13T05:51:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa28eb9c75864d979f7cd3c8bc3b9e02_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477116608/Put/seqid=0 2024-11-13T05:51:57,498 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into 760f2dce4cc34ba892a38644d6cd17fa(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:57,498 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:57,499 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477117454; duration=0sec 2024-11-13T05:51:57,499 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:57,499 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:51:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742205_1381 (size=12254) 2024-11-13T05:51:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:57,502 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:51:57,503 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/ab0c0711212148219120a689b4c75567 is 50, key is test_row_0/C:col10/1731477115884/Put/seqid=0 2024-11-13T05:51:57,505 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa28eb9c75864d979f7cd3c8bc3b9e02_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa28eb9c75864d979f7cd3c8bc3b9e02_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:57,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/79ff1c76939c49c884b76d6cd4a6c84d, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:57,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/79ff1c76939c49c884b76d6cd4a6c84d is 175, key is test_row_0/A:col10/1731477116608/Put/seqid=0 2024-11-13T05:51:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742207_1383 (size=31055) 2024-11-13T05:51:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742206_1382 (size=12309) 2024-11-13T05:51:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:57,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T05:51:57,912 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/79ff1c76939c49c884b76d6cd4a6c84d 2024-11-13T05:51:57,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9a674949a1114612a09a8f8ffaecff9a is 50, key is test_row_0/B:col10/1731477116608/Put/seqid=0 2024-11-13T05:51:57,924 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/ab0c0711212148219120a689b4c75567 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ab0c0711212148219120a689b4c75567 2024-11-13T05:51:57,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742208_1384 (size=12101) 2024-11-13T05:51:57,929 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into ab0c0711212148219120a689b4c75567(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:51:57,929 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:57,929 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477117454; duration=0sec 2024-11-13T05:51:57,930 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:51:57,930 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:51:57,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477177929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477178051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477178261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,328 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9a674949a1114612a09a8f8ffaecff9a 2024-11-13T05:51:58,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/f007a36f7afe47d1a01518e141d2ed5c is 50, key is test_row_0/C:col10/1731477116608/Put/seqid=0 2024-11-13T05:51:58,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742209_1385 (size=12101) 2024-11-13T05:51:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477178581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477178721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,729 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477178732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477178732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,739 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,739 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,742 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/f007a36f7afe47d1a01518e141d2ed5c 2024-11-13T05:51:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477178741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:58,745 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:51:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/79ff1c76939c49c884b76d6cd4a6c84d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d 2024-11-13T05:51:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:58,749 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d, entries=150, sequenceid=139, filesize=30.3 K 2024-11-13T05:51:58,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9a674949a1114612a09a8f8ffaecff9a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a674949a1114612a09a8f8ffaecff9a 2024-11-13T05:51:58,753 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a674949a1114612a09a8f8ffaecff9a, entries=150, sequenceid=139, filesize=11.8 K 2024-11-13T05:51:58,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/f007a36f7afe47d1a01518e141d2ed5c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f007a36f7afe47d1a01518e141d2ed5c 2024-11-13T05:51:58,758 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f007a36f7afe47d1a01518e141d2ed5c, entries=150, sequenceid=139, filesize=11.8 K 2024-11-13T05:51:58,759 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1270ms, sequenceid=139, compaction requested=false 2024-11-13T05:51:58,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:58,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:58,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T05:51:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T05:51:58,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T05:51:58,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-11-13T05:51:58,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.0400 sec 2024-11-13T05:51:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T05:51:58,871 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:51:58,872 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:51:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T05:51:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:51:58,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:51:58,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:51:58,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:51:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:51:59,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T05:51:59,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:59,028 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:51:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:51:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:51:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:51:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:51:59,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e4a9b15a24bf4c64998ec2fd9c09526b_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477117900/Put/seqid=0 2024-11-13T05:51:59,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742210_1386 (size=12304) 2024-11-13T05:51:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:59,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:51:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:51:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477179212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:59,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477179331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:59,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:51:59,484 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e4a9b15a24bf4c64998ec2fd9c09526b_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e4a9b15a24bf4c64998ec2fd9c09526b_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:51:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/881236a576ed4c43be8eaa0c5dd3bf25, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:51:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/881236a576ed4c43be8eaa0c5dd3bf25 is 175, key is test_row_0/A:col10/1731477117900/Put/seqid=0 2024-11-13T05:51:59,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742211_1387 (size=31105) 2024-11-13T05:51:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:51:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477179541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:51:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477179861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:51:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:51:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:51:59,890 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/881236a576ed4c43be8eaa0c5dd3bf25 2024-11-13T05:51:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/1c32e110dbf44e4c9f63fc13837bb8a8 is 50, key is test_row_0/B:col10/1731477117900/Put/seqid=0 2024-11-13T05:51:59,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742212_1388 (size=12151) 2024-11-13T05:51:59,944 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/1c32e110dbf44e4c9f63fc13837bb8a8 2024-11-13T05:51:59,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/65fedb526d8f4aa0bd9a9da52938bb63 is 50, key is test_row_0/C:col10/1731477117900/Put/seqid=0 2024-11-13T05:51:59,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742213_1389 (size=12151) 2024-11-13T05:51:59,958 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/65fedb526d8f4aa0bd9a9da52938bb63 2024-11-13T05:51:59,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/881236a576ed4c43be8eaa0c5dd3bf25 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25 2024-11-13T05:51:59,972 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25, entries=150, sequenceid=157, filesize=30.4 K 2024-11-13T05:51:59,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/1c32e110dbf44e4c9f63fc13837bb8a8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/1c32e110dbf44e4c9f63fc13837bb8a8 2024-11-13T05:51:59,978 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/1c32e110dbf44e4c9f63fc13837bb8a8, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T05:51:59,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/65fedb526d8f4aa0bd9a9da52938bb63 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/65fedb526d8f4aa0bd9a9da52938bb63 2024-11-13T05:51:59,982 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/65fedb526d8f4aa0bd9a9da52938bb63, entries=150, sequenceid=157, filesize=11.9 K 2024-11-13T05:51:59,983 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 49578a99aa8fa6a60ec17a19ef73c0fd in 955ms, sequenceid=157, compaction requested=true 2024-11-13T05:51:59,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:51:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:51:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T05:51:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T05:51:59,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T05:51:59,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1130 sec 2024-11-13T05:51:59,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.1180 sec 2024-11-13T05:52:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T05:52:00,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:00,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T05:52:00,014 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:52:00,014 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:00,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:52:00,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T05:52:00,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:00,166 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:52:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:00,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d7bce21e91c04cb3a9119314add0bc35_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477119207/Put/seqid=0 2024-11-13T05:52:00,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742214_1390 (size=12304) 2024-11-13T05:52:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:00,181 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d7bce21e91c04cb3a9119314add0bc35_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d7bce21e91c04cb3a9119314add0bc35_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:00,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/34576a54c6a340ba9d4e0518f76ae474, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:00,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/34576a54c6a340ba9d4e0518f76ae474 is 175, key is test_row_0/A:col10/1731477119207/Put/seqid=0 2024-11-13T05:52:00,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742215_1391 (size=31105) 2024-11-13T05:52:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:52:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:00,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477180467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:00,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477180581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:00,585 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/34576a54c6a340ba9d4e0518f76ae474 2024-11-13T05:52:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/db1bd15af86f4456b9e651ebac893765 is 50, key is test_row_0/B:col10/1731477119207/Put/seqid=0 2024-11-13T05:52:00,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742216_1392 (size=12151) 2024-11-13T05:52:00,595 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/db1bd15af86f4456b9e651ebac893765 2024-11-13T05:52:00,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/808482f01cdb44b28e2176f7ace9f115 is 50, key is test_row_0/C:col10/1731477119207/Put/seqid=0 2024-11-13T05:52:00,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742217_1393 (size=12151) 2024-11-13T05:52:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:52:00,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477180792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:01,005 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/808482f01cdb44b28e2176f7ace9f115 2024-11-13T05:52:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/34576a54c6a340ba9d4e0518f76ae474 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474 2024-11-13T05:52:01,011 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474, entries=150, sequenceid=174, filesize=30.4 K 2024-11-13T05:52:01,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/db1bd15af86f4456b9e651ebac893765 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/db1bd15af86f4456b9e651ebac893765 2024-11-13T05:52:01,019 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/db1bd15af86f4456b9e651ebac893765, entries=150, sequenceid=174, filesize=11.9 K 2024-11-13T05:52:01,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/808482f01cdb44b28e2176f7ace9f115 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/808482f01cdb44b28e2176f7ace9f115 2024-11-13T05:52:01,023 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/808482f01cdb44b28e2176f7ace9f115, entries=150, sequenceid=174, filesize=11.9 K 2024-11-13T05:52:01,024 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 49578a99aa8fa6a60ec17a19ef73c0fd in 858ms, sequenceid=174, compaction requested=true 2024-11-13T05:52:01,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:01,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T05:52:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T05:52:01,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T05:52:01,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-13T05:52:01,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0150 sec 2024-11-13T05:52:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:01,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:52:01,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:01,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:01,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:01,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:01,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:01,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:01,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fccfe764ea594d58803054c8ac70e20a_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:01,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742218_1394 (size=14794) 2024-11-13T05:52:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T05:52:01,152 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:01,152 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-13T05:52:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:01,154 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:01,154 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:01,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:01,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477181230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:01,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477181351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:01,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:01,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:01,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fccfe764ea594d58803054c8ac70e20a_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fccfe764ea594d58803054c8ac70e20a_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:01,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/de79487e6f9a44cfa778dfe436739b08, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:01,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/de79487e6f9a44cfa778dfe436739b08 is 175, key is test_row_0/A:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:01,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742219_1395 (size=39749) 2024-11-13T05:52:01,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477181571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:01,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:01,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477181892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:01,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:01,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:01,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/de79487e6f9a44cfa778dfe436739b08 2024-11-13T05:52:01,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/dd69e7c5061c445e8b0f5bb8394b08c7 is 50, key is test_row_0/B:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742220_1396 (size=12151) 2024-11-13T05:52:02,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:02,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/dd69e7c5061c445e8b0f5bb8394b08c7 2024-11-13T05:52:02,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/ffe2c8c8bd064e55878c337ce13220ba is 50, key is test_row_0/C:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:02,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742221_1397 (size=12151) 2024-11-13T05:52:02,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477182412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:02,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/ffe2c8c8bd064e55878c337ce13220ba 2024-11-13T05:52:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731477182751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:02,761 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:02,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/de79487e6f9a44cfa778dfe436739b08 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08 2024-11-13T05:52:02,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08, entries=200, sequenceid=190, filesize=38.8 K 2024-11-13T05:52:02,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/dd69e7c5061c445e8b0f5bb8394b08c7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/dd69e7c5061c445e8b0f5bb8394b08c7 2024-11-13T05:52:02,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/dd69e7c5061c445e8b0f5bb8394b08c7, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T05:52:02,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/ffe2c8c8bd064e55878c337ce13220ba as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ffe2c8c8bd064e55878c337ce13220ba 2024-11-13T05:52:02,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35824 deadline: 1731477182761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:02,773 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:02,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ffe2c8c8bd064e55878c337ce13220ba, entries=150, sequenceid=190, filesize=11.9 K 2024-11-13T05:52:02,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35852 deadline: 1731477182772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:02,781 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:02,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1666ms, sequenceid=190, compaction requested=true 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:02,782 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:02,782 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:02,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:02,783 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60863 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:52:02,783 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164277 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:52:02,783 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:52:02,783 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:52:02,783 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,784 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,784 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/cd6dd74614b04807af9061792c6be342, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a674949a1114612a09a8f8ffaecff9a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/1c32e110dbf44e4c9f63fc13837bb8a8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/db1bd15af86f4456b9e651ebac893765, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/dd69e7c5061c445e8b0f5bb8394b08c7] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=59.4 K 2024-11-13T05:52:02,784 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/760f2dce4cc34ba892a38644d6cd17fa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=160.4 K 2024-11-13T05:52:02,784 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/760f2dce4cc34ba892a38644d6cd17fa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08] 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting cd6dd74614b04807af9061792c6be342, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731477115884 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760f2dce4cc34ba892a38644d6cd17fa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731477115884 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a674949a1114612a09a8f8ffaecff9a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731477116608 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ff1c76939c49c884b76d6cd4a6c84d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731477116608 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c32e110dbf44e4c9f63fc13837bb8a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731477117900 2024-11-13T05:52:02,784 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 881236a576ed4c43be8eaa0c5dd3bf25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731477117900 2024-11-13T05:52:02,785 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting db1bd15af86f4456b9e651ebac893765, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731477119176 2024-11-13T05:52:02,785 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34576a54c6a340ba9d4e0518f76ae474, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731477119176 2024-11-13T05:52:02,785 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting dd69e7c5061c445e8b0f5bb8394b08c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731477120409 2024-11-13T05:52:02,785 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting de79487e6f9a44cfa778dfe436739b08, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731477120409 2024-11-13T05:52:02,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:52:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:02,793 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:02,794 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:02,794 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/d2323a6984e94646b980f779c2092c67 is 50, key is test_row_0/B:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:02,804 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411135a10118b286447c19324c6034b6adb42_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:02,807 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411135a10118b286447c19324c6034b6adb42_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:02,808 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411135a10118b286447c19324c6034b6adb42_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:02,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113948b07ab28d64d9eba924085144ea80a_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477121163/Put/seqid=0 2024-11-13T05:52:02,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742222_1398 (size=12629) 2024-11-13T05:52:02,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742223_1399 (size=4469) 2024-11-13T05:52:02,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742224_1400 (size=14794) 2024-11-13T05:52:02,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477182924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:02,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:02,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477183041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:03,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,215 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/d2323a6984e94646b980f779c2092c67 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d2323a6984e94646b980f779c2092c67 2024-11-13T05:52:03,219 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into d2323a6984e94646b980f779c2092c67(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:03,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:03,219 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=11, startTime=1731477122782; duration=0sec 2024-11-13T05:52:03,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:03,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:52:03,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T05:52:03,220 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:03,220 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/5926fdc613ec49ba935f3042228eacad is 175, key is test_row_0/A:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:03,221 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60863 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T05:52:03,221 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:52:03,221 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,221 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ab0c0711212148219120a689b4c75567, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f007a36f7afe47d1a01518e141d2ed5c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/65fedb526d8f4aa0bd9a9da52938bb63, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/808482f01cdb44b28e2176f7ace9f115, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ffe2c8c8bd064e55878c337ce13220ba] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=59.4 K 2024-11-13T05:52:03,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0c0711212148219120a689b4c75567, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731477115884 2024-11-13T05:52:03,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f007a36f7afe47d1a01518e141d2ed5c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731477116608 2024-11-13T05:52:03,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fedb526d8f4aa0bd9a9da52938bb63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731477117900 2024-11-13T05:52:03,223 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 808482f01cdb44b28e2176f7ace9f115, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731477119176 2024-11-13T05:52:03,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742225_1401 (size=31583) 2024-11-13T05:52:03,223 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe2c8c8bd064e55878c337ce13220ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731477120409 2024-11-13T05:52:03,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:03,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113948b07ab28d64d9eba924085144ea80a_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113948b07ab28d64d9eba924085144ea80a_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:03,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/992ad885b7ff447bb7bfdad721ed865a, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:03,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/992ad885b7ff447bb7bfdad721ed865a is 175, key is test_row_0/A:col10/1731477121163/Put/seqid=0 2024-11-13T05:52:03,232 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#336 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:03,232 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/9d5e35d49b5a47e88b4f759c810d479b is 50, key is test_row_0/C:col10/1731477120454/Put/seqid=0 2024-11-13T05:52:03,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742226_1402 (size=39749) 2024-11-13T05:52:03,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742227_1403 (size=12629) 2024-11-13T05:52:03,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477183262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:03,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477183431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:03,443 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:03,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477183581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:03,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,627 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/5926fdc613ec49ba935f3042228eacad as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5926fdc613ec49ba935f3042228eacad 2024-11-13T05:52:03,631 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into 5926fdc613ec49ba935f3042228eacad(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:03,631 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:03,631 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=11, startTime=1731477122782; duration=0sec 2024-11-13T05:52:03,631 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:03,631 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:52:03,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/992ad885b7ff447bb7bfdad721ed865a 2024-11-13T05:52:03,638 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/9d5e35d49b5a47e88b4f759c810d479b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/9d5e35d49b5a47e88b4f759c810d479b 2024-11-13T05:52:03,642 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into 9d5e35d49b5a47e88b4f759c810d479b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:03,642 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:03,642 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=11, startTime=1731477122782; duration=0sec 2024-11-13T05:52:03,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/a45433149d584c8baedd0c4855db7f85 is 50, key is test_row_0/B:col10/1731477121163/Put/seqid=0 2024-11-13T05:52:03,642 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:03,642 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:52:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742228_1404 (size=12151) 2024-11-13T05:52:03,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/a45433149d584c8baedd0c4855db7f85 2024-11-13T05:52:03,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/4895169c3e2049359f351e72a38434cc is 50, key is test_row_0/C:col10/1731477121163/Put/seqid=0 2024-11-13T05:52:03,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742229_1405 (size=12151) 2024-11-13T05:52:03,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:03,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/4895169c3e2049359f351e72a38434cc 2024-11-13T05:52:04,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/992ad885b7ff447bb7bfdad721ed865a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a 2024-11-13T05:52:04,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a, entries=200, sequenceid=207, filesize=38.8 K 2024-11-13T05:52:04,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/a45433149d584c8baedd0c4855db7f85 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a45433149d584c8baedd0c4855db7f85 2024-11-13T05:52:04,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a45433149d584c8baedd0c4855db7f85, entries=150, sequenceid=207, filesize=11.9 K 2024-11-13T05:52:04,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/4895169c3e2049359f351e72a38434cc as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4895169c3e2049359f351e72a38434cc 2024-11-13T05:52:04,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4895169c3e2049359f351e72a38434cc, entries=150, sequenceid=207, filesize=11.9 K 2024-11-13T05:52:04,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1277ms, sequenceid=207, compaction requested=false 2024-11-13T05:52:04,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:04,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:52:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:04,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:04,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cde3b646043240a69c1757ab200d464e_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:04,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742230_1406 (size=14794) 2024-11-13T05:52:04,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477184202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:04,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477184311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:04,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,520 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:04,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cde3b646043240a69c1757ab200d464e_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cde3b646043240a69c1757ab200d464e_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:04,524 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/43aace89ddcd41b5836efb59318f3f57, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:04,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/43aace89ddcd41b5836efb59318f3f57 is 175, key is test_row_0/A:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:04,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742231_1407 (size=39749) 2024-11-13T05:52:04,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477184522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:04,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477184842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:04,930 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/43aace89ddcd41b5836efb59318f3f57 2024-11-13T05:52:04,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/89a778ffb6ea4cffbb1505532460d15b is 50, key is test_row_0/B:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:04,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742232_1408 (size=12151) 2024-11-13T05:52:04,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:04,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:05,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:05,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:05,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:05,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/89a778ffb6ea4cffbb1505532460d15b 2024-11-13T05:52:05,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/36ae9e5f443240ccbced3985729e800d is 50, key is test_row_0/C:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:05,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477185362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:05,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742233_1409 (size=12151) 2024-11-13T05:52:05,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/36ae9e5f443240ccbced3985729e800d 2024-11-13T05:52:05,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/43aace89ddcd41b5836efb59318f3f57 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57 2024-11-13T05:52:05,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57, entries=200, sequenceid=226, filesize=38.8 K 2024-11-13T05:52:05,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/89a778ffb6ea4cffbb1505532460d15b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/89a778ffb6ea4cffbb1505532460d15b 2024-11-13T05:52:05,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/89a778ffb6ea4cffbb1505532460d15b, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T05:52:05,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/36ae9e5f443240ccbced3985729e800d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/36ae9e5f443240ccbced3985729e800d 2024-11-13T05:52:05,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/36ae9e5f443240ccbced3985729e800d, entries=150, sequenceid=226, filesize=11.9 K 2024-11-13T05:52:05,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1323ms, sequenceid=226, compaction requested=true 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:05,431 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:05,431 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:05,432 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:05,432 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:52:05,432 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,432 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d2323a6984e94646b980f779c2092c67, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a45433149d584c8baedd0c4855db7f85, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/89a778ffb6ea4cffbb1505532460d15b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.1 K 2024-11-13T05:52:05,432 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:05,432 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:52:05,432 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,432 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5926fdc613ec49ba935f3042228eacad, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=108.5 K 2024-11-13T05:52:05,433 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,433 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5926fdc613ec49ba935f3042228eacad, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57] 2024-11-13T05:52:05,433 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d2323a6984e94646b980f779c2092c67, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731477120409 2024-11-13T05:52:05,433 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a45433149d584c8baedd0c4855db7f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731477121163 2024-11-13T05:52:05,433 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5926fdc613ec49ba935f3042228eacad, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731477120409 2024-11-13T05:52:05,434 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 89a778ffb6ea4cffbb1505532460d15b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731477122880 2024-11-13T05:52:05,434 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 992ad885b7ff447bb7bfdad721ed865a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731477121163 2024-11-13T05:52:05,435 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43aace89ddcd41b5836efb59318f3f57, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731477122880 2024-11-13T05:52:05,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,451 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:05,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:05,453 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:05,453 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9b38bd93a9b140c496cd2756cdbc7bb7 is 50, key is test_row_0/B:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:05,463 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:05,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. as already flushing 2024-11-13T05:52:05,483 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136643074d47ac4eb9bab8a1ecff1a841c_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:05,485 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136643074d47ac4eb9bab8a1ecff1a841c_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:05,485 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136643074d47ac4eb9bab8a1ecff1a841c_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:05,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111347794978d8604089b460af310fe5f691_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477124189/Put/seqid=0 2024-11-13T05:52:05,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742235_1411 (size=4469) 2024-11-13T05:52:05,560 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#343 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:05,560 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/33d385d2f402414ab20eccd42149467f is 175, key is test_row_0/A:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:05,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742234_1410 (size=12731) 2024-11-13T05:52:05,580 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9b38bd93a9b140c496cd2756cdbc7bb7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9b38bd93a9b140c496cd2756cdbc7bb7 2024-11-13T05:52:05,589 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into 9b38bd93a9b140c496cd2756cdbc7bb7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:05,589 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:05,589 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477125431; duration=0sec 2024-11-13T05:52:05,589 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:05,589 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:52:05,589 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:05,590 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:05,590 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:52:05,590 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,591 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/9d5e35d49b5a47e88b4f759c810d479b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4895169c3e2049359f351e72a38434cc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/36ae9e5f443240ccbced3985729e800d] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.1 K 2024-11-13T05:52:05,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742236_1412 (size=12304) 2024-11-13T05:52:05,591 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5e35d49b5a47e88b4f759c810d479b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731477120409 2024-11-13T05:52:05,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:05,592 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4895169c3e2049359f351e72a38434cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731477121163 2024-11-13T05:52:05,592 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ae9e5f443240ccbced3985729e800d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731477122880 2024-11-13T05:52:05,598 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111347794978d8604089b460af310fe5f691_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347794978d8604089b460af310fe5f691_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:05,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87d3428c20fd4a889be4af2fc53548f7, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:05,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87d3428c20fd4a889be4af2fc53548f7 is 175, key is test_row_0/A:col10/1731477124189/Put/seqid=0 2024-11-13T05:52:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742237_1413 (size=31685) 2024-11-13T05:52:05,625 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/33d385d2f402414ab20eccd42149467f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/33d385d2f402414ab20eccd42149467f 2024-11-13T05:52:05,630 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into 33d385d2f402414ab20eccd42149467f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:05,630 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:05,630 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477125431; duration=0sec 2024-11-13T05:52:05,630 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:05,630 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:52:05,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477185621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:05,639 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:05,640 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/f9c0992b22f04089b7b5c6e2ea611b00 is 50, key is test_row_0/C:col10/1731477124106/Put/seqid=0 2024-11-13T05:52:05,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742238_1414 (size=31105) 2024-11-13T05:52:05,643 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87d3428c20fd4a889be4af2fc53548f7 2024-11-13T05:52:05,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9a895683683a4ee2ba5bc824b3330028 is 50, key is test_row_0/B:col10/1731477124189/Put/seqid=0 2024-11-13T05:52:05,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742239_1415 (size=12731) 2024-11-13T05:52:05,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742240_1416 (size=12151) 2024-11-13T05:52:05,716 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9a895683683a4ee2ba5bc824b3330028 2024-11-13T05:52:05,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/4938153051904254ba41691c8b34db73 is 50, key is test_row_0/C:col10/1731477124189/Put/seqid=0 2024-11-13T05:52:05,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477185742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:05,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742241_1417 (size=12151) 2024-11-13T05:52:05,760 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/4938153051904254ba41691c8b34db73 2024-11-13T05:52:05,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/87d3428c20fd4a889be4af2fc53548f7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7 2024-11-13T05:52:05,775 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7, entries=150, sequenceid=242, filesize=30.4 K 2024-11-13T05:52:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/9a895683683a4ee2ba5bc824b3330028 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a895683683a4ee2ba5bc824b3330028 2024-11-13T05:52:05,784 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a895683683a4ee2ba5bc824b3330028, entries=150, sequenceid=242, filesize=11.9 K 2024-11-13T05:52:05,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/4938153051904254ba41691c8b34db73 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4938153051904254ba41691c8b34db73 2024-11-13T05:52:05,788 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4938153051904254ba41691c8b34db73, entries=150, sequenceid=242, filesize=11.9 K 2024-11-13T05:52:05,789 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49578a99aa8fa6a60ec17a19ef73c0fd in 338ms, sequenceid=242, compaction requested=false 2024-11-13T05:52:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-13T05:52:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-13T05:52:05,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T05:52:05,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.6370 sec 2024-11-13T05:52:05,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 4.6420 sec 2024-11-13T05:52:05,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:52:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:05,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113db90b59e3d654c3b9dd8bea04c05d623_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742242_1418 (size=14894) 2024-11-13T05:52:06,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:06,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113db90b59e3d654c3b9dd8bea04c05d623_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113db90b59e3d654c3b9dd8bea04c05d623_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:06,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/af1d86799d644a069fd5cd06b2405ca8, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:06,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/af1d86799d644a069fd5cd06b2405ca8 is 175, key is test_row_0/A:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742243_1419 (size=39849) 2024-11-13T05:52:06,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/af1d86799d644a069fd5cd06b2405ca8 2024-11-13T05:52:06,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/ea956b2f17c144a1a8b3b7ed2c5e60ea is 50, key is test_row_0/B:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742244_1420 (size=12251) 2024-11-13T05:52:06,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/ea956b2f17c144a1a8b3b7ed2c5e60ea 2024-11-13T05:52:06,091 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/f9c0992b22f04089b7b5c6e2ea611b00 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f9c0992b22f04089b7b5c6e2ea611b00 2024-11-13T05:52:06,097 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into f9c0992b22f04089b7b5c6e2ea611b00(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:06,097 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:06,097 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477125431; duration=0sec 2024-11-13T05:52:06,097 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:06,097 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:52:06,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/20500e481da44a0689fd097c8b345937 is 50, key is test_row_0/C:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477186115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:06,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742245_1421 (size=12251) 2024-11-13T05:52:06,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/20500e481da44a0689fd097c8b345937 2024-11-13T05:52:06,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/af1d86799d644a069fd5cd06b2405ca8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8 2024-11-13T05:52:06,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8, entries=200, sequenceid=261, filesize=38.9 K 2024-11-13T05:52:06,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/ea956b2f17c144a1a8b3b7ed2c5e60ea as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ea956b2f17c144a1a8b3b7ed2c5e60ea 2024-11-13T05:52:06,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ea956b2f17c144a1a8b3b7ed2c5e60ea, entries=150, sequenceid=261, filesize=12.0 K 2024-11-13T05:52:06,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/20500e481da44a0689fd097c8b345937 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/20500e481da44a0689fd097c8b345937 2024-11-13T05:52:06,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/20500e481da44a0689fd097c8b345937, entries=150, sequenceid=261, filesize=12.0 K 2024-11-13T05:52:06,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 49578a99aa8fa6a60ec17a19ef73c0fd in 198ms, sequenceid=261, compaction requested=true 2024-11-13T05:52:06,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:06,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:06,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:06,164 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:06,164 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:06,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:06,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:06,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:06,165 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:06,165 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:52:06,165 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:06,165 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9b38bd93a9b140c496cd2756cdbc7bb7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a895683683a4ee2ba5bc824b3330028, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ea956b2f17c144a1a8b3b7ed2c5e60ea] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.3 K 2024-11-13T05:52:06,165 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:06,165 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:52:06,165 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:06,165 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/33d385d2f402414ab20eccd42149467f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=100.2 K 2024-11-13T05:52:06,166 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:06,166 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/33d385d2f402414ab20eccd42149467f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8] 2024-11-13T05:52:06,166 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b38bd93a9b140c496cd2756cdbc7bb7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731477122880 2024-11-13T05:52:06,166 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33d385d2f402414ab20eccd42149467f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731477122880 2024-11-13T05:52:06,166 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a895683683a4ee2ba5bc824b3330028, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731477124185 2024-11-13T05:52:06,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:06,166 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d3428c20fd4a889be4af2fc53548f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731477124185 2024-11-13T05:52:06,167 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ea956b2f17c144a1a8b3b7ed2c5e60ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477125514 2024-11-13T05:52:06,167 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting af1d86799d644a069fd5cd06b2405ca8, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477125514 2024-11-13T05:52:06,188 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:06,189 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/ba5acbcf1e7145f4b2af90d427f2ff78 is 50, key is test_row_0/B:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,203 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:06,209 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113a6f6fdcdabe142b3a2e255ad223d72fd_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:06,211 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113a6f6fdcdabe142b3a2e255ad223d72fd_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:06,211 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a6f6fdcdabe142b3a2e255ad223d72fd_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:06,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742246_1422 (size=12933) 2024-11-13T05:52:06,225 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/ba5acbcf1e7145f4b2af90d427f2ff78 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ba5acbcf1e7145f4b2af90d427f2ff78 2024-11-13T05:52:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:06,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:52:06,238 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into ba5acbcf1e7145f4b2af90d427f2ff78(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:06,238 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:06,238 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477126164; duration=0sec 2024-11-13T05:52:06,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:06,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:06,239 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:06,239 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:52:06,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:06,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:06,239 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:06,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:06,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:06,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742247_1423 (size=4469) 2024-11-13T05:52:06,240 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:06,241 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:52:06,241 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:06,241 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f9c0992b22f04089b7b5c6e2ea611b00, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4938153051904254ba41691c8b34db73, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/20500e481da44a0689fd097c8b345937] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.3 K 2024-11-13T05:52:06,241 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c0992b22f04089b7b5c6e2ea611b00, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731477122880 2024-11-13T05:52:06,241 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4938153051904254ba41691c8b34db73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731477124185 2024-11-13T05:52:06,242 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 20500e481da44a0689fd097c8b345937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477125514 2024-11-13T05:52:06,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113771e186e262449c6a947af66b74d7759_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477126029/Put/seqid=0 2024-11-13T05:52:06,261 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:06,261 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/8682e3587dea4b2b8c1a9040582aae9c is 50, key is test_row_0/C:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742248_1424 (size=14994) 2024-11-13T05:52:06,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:06,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113771e186e262449c6a947af66b74d7759_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113771e186e262449c6a947af66b74d7759_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:06,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/9d800bc5446a4800afc8ec71d4e6f42f, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:06,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/9d800bc5446a4800afc8ec71d4e6f42f is 175, key is test_row_0/A:col10/1731477126029/Put/seqid=0 2024-11-13T05:52:06,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742249_1425 (size=12933) 2024-11-13T05:52:06,283 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/8682e3587dea4b2b8c1a9040582aae9c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/8682e3587dea4b2b8c1a9040582aae9c 2024-11-13T05:52:06,286 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into 8682e3587dea4b2b8c1a9040582aae9c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:06,286 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:06,287 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477126165; duration=0sec 2024-11-13T05:52:06,287 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:06,287 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:52:06,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742250_1426 (size=39949) 2024-11-13T05:52:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477186343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:06,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477186381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:06,390 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 49578a99aa8fa6a60ec17a19ef73c0fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477186451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:06,642 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:06,642 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/68dc484290c042f1846e35a657cbfd84 is 175, key is test_row_0/A:col10/1731477125619/Put/seqid=0 2024-11-13T05:52:06,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742251_1427 (size=31887) 2024-11-13T05:52:06,651 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/68dc484290c042f1846e35a657cbfd84 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/68dc484290c042f1846e35a657cbfd84 2024-11-13T05:52:06,656 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into 68dc484290c042f1846e35a657cbfd84(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:06,656 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:06,656 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477126164; duration=0sec 2024-11-13T05:52:06,656 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:06,656 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:52:06,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477186671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:06,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:06,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/9d800bc5446a4800afc8ec71d4e6f42f 2024-11-13T05:52:06,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/4d6335d6477b4977ab56684c65f6dd7b is 50, key is test_row_0/B:col10/1731477126029/Put/seqid=0 2024-11-13T05:52:06,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742252_1428 (size=12301) 2024-11-13T05:52:06,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/4d6335d6477b4977ab56684c65f6dd7b 2024-11-13T05:52:06,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/5272c8638ee64070b89bb21161be1d5b is 50, key is test_row_0/C:col10/1731477126029/Put/seqid=0 2024-11-13T05:52:06,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742253_1429 (size=12301) 2024-11-13T05:52:06,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/5272c8638ee64070b89bb21161be1d5b 2024-11-13T05:52:06,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/9d800bc5446a4800afc8ec71d4e6f42f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f 2024-11-13T05:52:06,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f, entries=200, sequenceid=279, filesize=39.0 K 2024-11-13T05:52:06,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/4d6335d6477b4977ab56684c65f6dd7b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4d6335d6477b4977ab56684c65f6dd7b 2024-11-13T05:52:06,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4d6335d6477b4977ab56684c65f6dd7b, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T05:52:06,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/5272c8638ee64070b89bb21161be1d5b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5272c8638ee64070b89bb21161be1d5b 2024-11-13T05:52:06,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5272c8638ee64070b89bb21161be1d5b, entries=150, sequenceid=279, filesize=12.0 K 2024-11-13T05:52:06,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49578a99aa8fa6a60ec17a19ef73c0fd in 546ms, sequenceid=279, compaction requested=false 2024-11-13T05:52:06,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:06,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:52:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:06,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:07,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb8fef864cb84603882ea5ec8223357c_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:07,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742254_1430 (size=14994) 2024-11-13T05:52:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477187111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477187231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:07,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:07,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb8fef864cb84603882ea5ec8223357c_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bb8fef864cb84603882ea5ec8223357c_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:07,417 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/96e6b46300b4420f978d2452874367be, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:07,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/96e6b46300b4420f978d2452874367be is 175, key is test_row_0/A:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:07,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742255_1431 (size=39949) 2024-11-13T05:52:07,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477187441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477187761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:07,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/96e6b46300b4420f978d2452874367be 2024-11-13T05:52:07,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/a92d68ca8bc8439e803ca4c1adf337f1 is 50, key is test_row_0/B:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:07,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742256_1432 (size=12301) 2024-11-13T05:52:08,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/a92d68ca8bc8439e803ca4c1adf337f1 2024-11-13T05:52:08,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/06b04ec2ab044d72b514c8db99f43eab is 50, key is test_row_0/C:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:08,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742257_1433 (size=12301) 2024-11-13T05:52:08,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/06b04ec2ab044d72b514c8db99f43eab 2024-11-13T05:52:08,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/96e6b46300b4420f978d2452874367be as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be 2024-11-13T05:52:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35866 deadline: 1731477188282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:08,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be, entries=200, sequenceid=298, filesize=39.0 K 2024-11-13T05:52:08,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/a92d68ca8bc8439e803ca4c1adf337f1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a92d68ca8bc8439e803ca4c1adf337f1 2024-11-13T05:52:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a92d68ca8bc8439e803ca4c1adf337f1, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:52:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/06b04ec2ab044d72b514c8db99f43eab as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/06b04ec2ab044d72b514c8db99f43eab 2024-11-13T05:52:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/06b04ec2ab044d72b514c8db99f43eab, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T05:52:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1314ms, sequenceid=298, compaction requested=true 2024-11-13T05:52:08,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:08,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:08,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:08,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:08,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:08,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:08,309 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:08,309 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:08,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111785 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:08,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:52:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,310 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:08,310 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/68dc484290c042f1846e35a657cbfd84, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=109.2 K 2024-11-13T05:52:08,310 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:08,310 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/68dc484290c042f1846e35a657cbfd84, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be] 2024-11-13T05:52:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,310 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:08,310 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:52:08,310 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:08,311 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ba5acbcf1e7145f4b2af90d427f2ff78, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4d6335d6477b4977ab56684c65f6dd7b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a92d68ca8bc8439e803ca4c1adf337f1] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.7 K 2024-11-13T05:52:08,311 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5acbcf1e7145f4b2af90d427f2ff78, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477125514 2024-11-13T05:52:08,311 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68dc484290c042f1846e35a657cbfd84, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477125514 2024-11-13T05:52:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,312 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d6335d6477b4977ab56684c65f6dd7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731477126029 2024-11-13T05:52:08,312 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d800bc5446a4800afc8ec71d4e6f42f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731477126029 2024-11-13T05:52:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,312 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a92d68ca8bc8439e803ca4c1adf337f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477126328 2024-11-13T05:52:08,312 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e6b46300b4420f978d2452874367be, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477126328 2024-11-13T05:52:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,327 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,332 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,332 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/29765679f2cf49e49a8bf2f8c822f86c is 50, key is test_row_0/B:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,333 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111357a9da9d6c01491faee906c6354b8630_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,340 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111357a9da9d6c01491faee906c6354b8630_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:08,340 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111357a9da9d6c01491faee906c6354b8630_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:08,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742258_1434 (size=13085) 2024-11-13T05:52:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742259_1435 (size=4469) 2024-11-13T05:52:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#360 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/f56422f53f1748b49211d718f8be3d48 is 175, key is test_row_0/A:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742260_1436 (size=32039) 2024-11-13T05:52:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:52:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:08,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113056910ad3c3b45689b5c087ae29b4ca8_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477127062/Put/seqid=0 2024-11-13T05:52:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742261_1437 (size=12454) 2024-11-13T05:52:08,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113056910ad3c3b45689b5c087ae29b4ca8_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113056910ad3c3b45689b5c087ae29b4ca8_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:08,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0c553f7648064faf84ddb96eb06564d9, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:08,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0c553f7648064faf84ddb96eb06564d9 is 175, key is test_row_0/A:col10/1731477127062/Put/seqid=0 2024-11-13T05:52:08,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742262_1438 (size=31255) 2024-11-13T05:52:08,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477188563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:08,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477188691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:08,752 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/29765679f2cf49e49a8bf2f8c822f86c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/29765679f2cf49e49a8bf2f8c822f86c 2024-11-13T05:52:08,759 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into 29765679f2cf49e49a8bf2f8c822f86c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:08,759 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:08,759 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477128308; duration=0sec 2024-11-13T05:52:08,759 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:08,759 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:52:08,759 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:08,762 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:08,762 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:52:08,762 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:08,762 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/8682e3587dea4b2b8c1a9040582aae9c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5272c8638ee64070b89bb21161be1d5b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/06b04ec2ab044d72b514c8db99f43eab] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.7 K 2024-11-13T05:52:08,763 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8682e3587dea4b2b8c1a9040582aae9c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731477125514 2024-11-13T05:52:08,763 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5272c8638ee64070b89bb21161be1d5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731477126029 2024-11-13T05:52:08,763 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b04ec2ab044d72b514c8db99f43eab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477126328 2024-11-13T05:52:08,774 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:08,774 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a1a297b807534368bf79935e188ac80b is 50, key is test_row_0/C:col10/1731477126331/Put/seqid=0 2024-11-13T05:52:08,794 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/f56422f53f1748b49211d718f8be3d48 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/f56422f53f1748b49211d718f8be3d48 2024-11-13T05:52:08,800 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into f56422f53f1748b49211d718f8be3d48(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:08,801 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:08,801 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477128308; duration=0sec 2024-11-13T05:52:08,801 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:08,801 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:52:08,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742263_1439 (size=13085) 2024-11-13T05:52:08,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0c553f7648064faf84ddb96eb06564d9 2024-11-13T05:52:08,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/b6d936f4696d4df7893d163e82fba8a2 is 50, key is test_row_0/B:col10/1731477127062/Put/seqid=0 2024-11-13T05:52:08,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742264_1440 (size=12301) 2024-11-13T05:52:08,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/b6d936f4696d4df7893d163e82fba8a2 2024-11-13T05:52:08,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/3cc7343461d641479a69c35bbf2f3326 is 50, key is test_row_0/C:col10/1731477127062/Put/seqid=0 2024-11-13T05:52:08,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35836 deadline: 1731477188911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:08,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742265_1441 (size=12301) 2024-11-13T05:52:08,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/3cc7343461d641479a69c35bbf2f3326 2024-11-13T05:52:08,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/0c553f7648064faf84ddb96eb06564d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9 2024-11-13T05:52:08,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9, entries=150, sequenceid=314, filesize=30.5 K 2024-11-13T05:52:08,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/b6d936f4696d4df7893d163e82fba8a2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/b6d936f4696d4df7893d163e82fba8a2 2024-11-13T05:52:08,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/b6d936f4696d4df7893d163e82fba8a2, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T05:52:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/3cc7343461d641479a69c35bbf2f3326 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/3cc7343461d641479a69c35bbf2f3326 2024-11-13T05:52:08,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/3cc7343461d641479a69c35bbf2f3326, entries=150, sequenceid=314, filesize=12.0 K 2024-11-13T05:52:08,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49578a99aa8fa6a60ec17a19ef73c0fd in 569ms, sequenceid=314, compaction requested=false 2024-11-13T05:52:08,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:09,216 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/a1a297b807534368bf79935e188ac80b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a1a297b807534368bf79935e188ac80b 2024-11-13T05:52:09,222 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into a1a297b807534368bf79935e188ac80b(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:09,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:09,222 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477128309; duration=0sec 2024-11-13T05:52:09,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:09,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:52:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:09,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:52:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:09,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:09,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c50a1c634cc4ec287113ff98e12c54d_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:09,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742266_1442 (size=14994) 2024-11-13T05:52:09,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:09,257 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139c50a1c634cc4ec287113ff98e12c54d_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c50a1c634cc4ec287113ff98e12c54d_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:09,258 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/5cb2b706e2014aa695c0e162e9414809, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:09,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/5cb2b706e2014aa695c0e162e9414809 is 175, key is test_row_0/A:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742267_1443 (size=39949) 2024-11-13T05:52:09,272 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-13T05:52:09,272 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,272 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,272 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,274 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-13T05:52:09,274 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,274 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,274 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,274 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-11-13T05:52:09,274 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,274 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,275 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,279 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-13T05:52:09,279 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,279 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,279 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,285 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-13T05:52:09,285 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,285 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,285 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,286 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-13T05:52:09,286 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,286 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,286 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,303 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-13T05:52:09,303 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:09,303 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,303 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:09,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T05:52:09,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:09,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/5cb2b706e2014aa695c0e162e9414809 2024-11-13T05:52:09,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/c3441010303d491f962605cd6f07aa20 is 50, key is test_row_0/B:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:09,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742268_1444 (size=12301) 2024-11-13T05:52:10,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/c3441010303d491f962605cd6f07aa20 2024-11-13T05:52:10,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/2c02846becb140019e9e5641a1540b9b is 50, key is test_row_0/C:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:10,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742269_1445 (size=12301) 2024-11-13T05:52:10,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/2c02846becb140019e9e5641a1540b9b 2024-11-13T05:52:10,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/5cb2b706e2014aa695c0e162e9414809 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809 2024-11-13T05:52:10,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809, entries=200, sequenceid=334, filesize=39.0 K 2024-11-13T05:52:10,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/c3441010303d491f962605cd6f07aa20 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/c3441010303d491f962605cd6f07aa20 2024-11-13T05:52:10,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/c3441010303d491f962605cd6f07aa20, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T05:52:10,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/2c02846becb140019e9e5641a1540b9b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2c02846becb140019e9e5641a1540b9b 2024-11-13T05:52:10,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2c02846becb140019e9e5641a1540b9b, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T05:52:10,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=46.96 KB/48090 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1279ms, sequenceid=334, compaction requested=true 2024-11-13T05:52:10,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:10,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:10,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:10,516 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:10,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:10,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:10,516 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:10,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49578a99aa8fa6a60ec17a19ef73c0fd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:10,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:10,516 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:10,516 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:10,516 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/B is initiating minor compaction (all files) 2024-11-13T05:52:10,516 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/A is initiating minor compaction (all files) 2024-11-13T05:52:10,517 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/B in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:10,517 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/A in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:10,517 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/f56422f53f1748b49211d718f8be3d48, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=100.8 K 2024-11-13T05:52:10,517 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/29765679f2cf49e49a8bf2f8c822f86c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/b6d936f4696d4df7893d163e82fba8a2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/c3441010303d491f962605cd6f07aa20] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.8 K 2024-11-13T05:52:10,517 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/f56422f53f1748b49211d718f8be3d48, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809] 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 29765679f2cf49e49a8bf2f8c822f86c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477126328 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56422f53f1748b49211d718f8be3d48, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477126328 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b6d936f4696d4df7893d163e82fba8a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731477127062 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c553f7648064faf84ddb96eb06564d9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731477127062 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting c3441010303d491f962605cd6f07aa20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731477128519 2024-11-13T05:52:10,517 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb2b706e2014aa695c0e162e9414809, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731477128519 2024-11-13T05:52:10,522 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:10,524 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:10,524 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113da431cd2f03746a58b83bf8183806a3d_49578a99aa8fa6a60ec17a19ef73c0fd store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:10,525 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/6f6d506e913746b5b39f212bd4e9337b is 50, key is test_row_0/B:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:10,530 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113da431cd2f03746a58b83bf8183806a3d_49578a99aa8fa6a60ec17a19ef73c0fd, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:10,531 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113da431cd2f03746a58b83bf8183806a3d_49578a99aa8fa6a60ec17a19ef73c0fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:10,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742270_1446 (size=13187) 2024-11-13T05:52:10,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742271_1447 (size=4469) 2024-11-13T05:52:10,937 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:10,938 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/a4643897b44a4578b2309ce9cdfb337a is 175, key is test_row_0/A:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:10,939 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/6f6d506e913746b5b39f212bd4e9337b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6f6d506e913746b5b39f212bd4e9337b 2024-11-13T05:52:10,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742272_1448 (size=32141) 2024-11-13T05:52:10,946 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/B of 49578a99aa8fa6a60ec17a19ef73c0fd into 6f6d506e913746b5b39f212bd4e9337b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:10,946 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:10,946 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/B, priority=13, startTime=1731477130516; duration=0sec 2024-11-13T05:52:10,946 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:10,946 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:B 2024-11-13T05:52:10,946 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:10,947 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:10,947 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 49578a99aa8fa6a60ec17a19ef73c0fd/C is initiating minor compaction (all files) 2024-11-13T05:52:10,947 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49578a99aa8fa6a60ec17a19ef73c0fd/C in TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:10,948 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a1a297b807534368bf79935e188ac80b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/3cc7343461d641479a69c35bbf2f3326, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2c02846becb140019e9e5641a1540b9b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp, totalSize=36.8 K 2024-11-13T05:52:10,948 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a1a297b807534368bf79935e188ac80b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731477126328 2024-11-13T05:52:10,948 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc7343461d641479a69c35bbf2f3326, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731477127062 2024-11-13T05:52:10,949 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c02846becb140019e9e5641a1540b9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731477128519 2024-11-13T05:52:10,957 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49578a99aa8fa6a60ec17a19ef73c0fd#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:10,957 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/e0d0394e4a014dbfb0c98e38b2c82203 is 50, key is test_row_0/C:col10/1731477128560/Put/seqid=0 2024-11-13T05:52:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742273_1449 (size=13187) 2024-11-13T05:52:11,347 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/a4643897b44a4578b2309ce9cdfb337a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a4643897b44a4578b2309ce9cdfb337a 2024-11-13T05:52:11,351 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/A of 49578a99aa8fa6a60ec17a19ef73c0fd into a4643897b44a4578b2309ce9cdfb337a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:11,351 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:11,351 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/A, priority=13, startTime=1731477130515; duration=0sec 2024-11-13T05:52:11,352 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:11,352 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:A 2024-11-13T05:52:11,366 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/e0d0394e4a014dbfb0c98e38b2c82203 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e0d0394e4a014dbfb0c98e38b2c82203 2024-11-13T05:52:11,372 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49578a99aa8fa6a60ec17a19ef73c0fd/C of 49578a99aa8fa6a60ec17a19ef73c0fd into e0d0394e4a014dbfb0c98e38b2c82203(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:11,373 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:11,373 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., storeName=49578a99aa8fa6a60ec17a19ef73c0fd/C, priority=13, startTime=1731477130516; duration=0sec 2024-11-13T05:52:11,373 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:11,373 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49578a99aa8fa6a60ec17a19ef73c0fd:C 2024-11-13T05:52:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:12,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-13T05:52:12,793 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-13T05:52:12,793 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:12,793 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:12,793 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:12,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:12,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:12,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:12,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:12,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:12,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f8714fd45f394c0c80d9148bac26f61d_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_0/A:col10/1731477129259/Put/seqid=0 2024-11-13T05:52:12,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742274_1450 (size=12454) 2024-11-13T05:52:12,844 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-13T05:52:12,844 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:12,844 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,844 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:12,893 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-11-13T05:52:12,893 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:12,893 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,893 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1073 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3219 rows 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3246 rows 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1085 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3252 rows 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1085 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3255 rows 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-13T05:52:12,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3237 rows 2024-11-13T05:52:12,893 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:52:12,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:12,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:12,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:12,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:52:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:12,898 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477132898"}]},"ts":"1731477132898"} 2024-11-13T05:52:12,899 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:52:12,899 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:52:12,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:52:12,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, UNASSIGN}] 2024-11-13T05:52:12,902 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, UNASSIGN 2024-11-13T05:52:12,903 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:12,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, UNASSIGN because future has completed 2024-11-13T05:52:12,904 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:52:12,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:52:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:13,056 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:13,057 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:52:13,057 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 49578a99aa8fa6a60ec17a19ef73c0fd, disabling compactions & flushes 2024-11-13T05:52:13,057 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:13,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:13,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f8714fd45f394c0c80d9148bac26f61d_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8714fd45f394c0c80d9148bac26f61d_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:13,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/89f4f6d639d4407ab9c8c7d7f8ad561a, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:13,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/89f4f6d639d4407ab9c8c7d7f8ad561a is 175, key is test_row_0/A:col10/1731477129259/Put/seqid=0 2024-11-13T05:52:13,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742275_1451 (size=31255) 2024-11-13T05:52:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:13,612 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/89f4f6d639d4407ab9c8c7d7f8ad561a 2024-11-13T05:52:13,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/3db51c9398574a9a9a2f7219f1e8bbb8 is 50, key is test_row_0/B:col10/1731477129259/Put/seqid=0 2024-11-13T05:52:13,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742276_1452 (size=12301) 2024-11-13T05:52:14,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/3db51c9398574a9a9a2f7219f1e8bbb8 2024-11-13T05:52:14,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/b6c414e18ef243658d3ee3dac6c9f5c2 is 50, key is test_row_0/C:col10/1731477129259/Put/seqid=0 2024-11-13T05:52:14,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742277_1453 (size=12301) 2024-11-13T05:52:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:14,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/b6c414e18ef243658d3ee3dac6c9f5c2 2024-11-13T05:52:14,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/89f4f6d639d4407ab9c8c7d7f8ad561a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/89f4f6d639d4407ab9c8c7d7f8ad561a 2024-11-13T05:52:14,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/89f4f6d639d4407ab9c8c7d7f8ad561a, entries=150, sequenceid=348, filesize=30.5 K 2024-11-13T05:52:14,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/3db51c9398574a9a9a2f7219f1e8bbb8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3db51c9398574a9a9a2f7219f1e8bbb8 2024-11-13T05:52:14,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3db51c9398574a9a9a2f7219f1e8bbb8, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T05:52:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/b6c414e18ef243658d3ee3dac6c9f5c2 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/b6c414e18ef243658d3ee3dac6c9f5c2 2024-11-13T05:52:14,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/b6c414e18ef243658d3ee3dac6c9f5c2, entries=150, sequenceid=348, filesize=12.0 K 2024-11-13T05:52:14,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1653ms, sequenceid=348, compaction requested=false 2024-11-13T05:52:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49578a99aa8fa6a60ec17a19ef73c0fd: 2024-11-13T05:52:14,447 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. after waiting 0 ms 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:14,447 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 49578a99aa8fa6a60ec17a19ef73c0fd 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=A 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=B 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49578a99aa8fa6a60ec17a19ef73c0fd, store=C 2024-11-13T05:52:14,447 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:14,451 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fe0ba38b7170468993e5f0514058e756_49578a99aa8fa6a60ec17a19ef73c0fd is 50, key is test_row_2/A:col10/1731477132891/Put/seqid=0 2024-11-13T05:52:14,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742278_1454 (size=7374) 2024-11-13T05:52:14,495 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:52:14,855 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,860 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fe0ba38b7170468993e5f0514058e756_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fe0ba38b7170468993e5f0514058e756_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:14,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/1f98d2a66b404039b64e398ee294b3de, store: [table=TestAcidGuarantees family=A region=49578a99aa8fa6a60ec17a19ef73c0fd] 2024-11-13T05:52:14,861 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/1f98d2a66b404039b64e398ee294b3de is 175, key is test_row_2/A:col10/1731477132891/Put/seqid=0 2024-11-13T05:52:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742279_1455 (size=13865) 2024-11-13T05:52:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:15,265 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/1f98d2a66b404039b64e398ee294b3de 2024-11-13T05:52:15,271 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/95fefdf83f604cedb9bbb5d1398ffb42 is 50, key is test_row_2/B:col10/1731477132891/Put/seqid=0 2024-11-13T05:52:15,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742280_1456 (size=7415) 2024-11-13T05:52:15,674 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/95fefdf83f604cedb9bbb5d1398ffb42 2024-11-13T05:52:15,684 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/184d96550ca04a8ab04c7b6046cbce6e is 50, key is test_row_2/C:col10/1731477132891/Put/seqid=0 2024-11-13T05:52:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742281_1457 (size=7415) 2024-11-13T05:52:16,089 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/184d96550ca04a8ab04c7b6046cbce6e 2024-11-13T05:52:16,098 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/A/1f98d2a66b404039b64e398ee294b3de as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/1f98d2a66b404039b64e398ee294b3de 2024-11-13T05:52:16,101 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/1f98d2a66b404039b64e398ee294b3de, entries=50, sequenceid=353, filesize=13.5 K 2024-11-13T05:52:16,102 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/B/95fefdf83f604cedb9bbb5d1398ffb42 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/95fefdf83f604cedb9bbb5d1398ffb42 2024-11-13T05:52:16,105 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/95fefdf83f604cedb9bbb5d1398ffb42, entries=50, sequenceid=353, filesize=7.2 K 2024-11-13T05:52:16,105 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/.tmp/C/184d96550ca04a8ab04c7b6046cbce6e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/184d96550ca04a8ab04c7b6046cbce6e 2024-11-13T05:52:16,108 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/184d96550ca04a8ab04c7b6046cbce6e, entries=50, sequenceid=353, filesize=7.2 K 2024-11-13T05:52:16,109 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1662ms, sequenceid=353, compaction requested=true 2024-11-13T05:52:16,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0a9a99700fc7486b9ad675ecbd4e18bc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a41a97e1cf6a4be1a89ac6720c6dc587, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/760f2dce4cc34ba892a38644d6cd17fa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5926fdc613ec49ba935f3042228eacad, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/33d385d2f402414ab20eccd42149467f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/68dc484290c042f1846e35a657cbfd84, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/f56422f53f1748b49211d718f8be3d48, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809] to archive 2024-11-13T05:52:16,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:52:16,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/fc6bc1fea53f4dba95eaa603fb72004f 2024-11-13T05:52:16,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/06d925024a2749a391a0c6f5e5fb2771 2024-11-13T05:52:16,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0b35b7a6198b4bec939437979723a681 2024-11-13T05:52:16,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0a9a99700fc7486b9ad675ecbd4e18bc to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0a9a99700fc7486b9ad675ecbd4e18bc 2024-11-13T05:52:16,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0e1c45197653426689035fc7d7e0dbd8 2024-11-13T05:52:16,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87175c1e26fb4973a866b787ca72e140 2024-11-13T05:52:16,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a41a97e1cf6a4be1a89ac6720c6dc587 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a41a97e1cf6a4be1a89ac6720c6dc587 2024-11-13T05:52:16,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/4e7c17905e1e4fd4800456b6cfcb9da2 2024-11-13T05:52:16,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/10cf68fc038e4e5ea99dee5f34e4c359 2024-11-13T05:52:16,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/760f2dce4cc34ba892a38644d6cd17fa to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/760f2dce4cc34ba892a38644d6cd17fa 2024-11-13T05:52:16,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/79ff1c76939c49c884b76d6cd4a6c84d 2024-11-13T05:52:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/881236a576ed4c43be8eaa0c5dd3bf25 2024-11-13T05:52:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/34576a54c6a340ba9d4e0518f76ae474 2024-11-13T05:52:16,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/de79487e6f9a44cfa778dfe436739b08 2024-11-13T05:52:16,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5926fdc613ec49ba935f3042228eacad to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5926fdc613ec49ba935f3042228eacad 2024-11-13T05:52:16,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/992ad885b7ff447bb7bfdad721ed865a 2024-11-13T05:52:16,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/43aace89ddcd41b5836efb59318f3f57 2024-11-13T05:52:16,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/33d385d2f402414ab20eccd42149467f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/33d385d2f402414ab20eccd42149467f 2024-11-13T05:52:16,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/87d3428c20fd4a889be4af2fc53548f7 2024-11-13T05:52:16,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/af1d86799d644a069fd5cd06b2405ca8 2024-11-13T05:52:16,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/68dc484290c042f1846e35a657cbfd84 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/68dc484290c042f1846e35a657cbfd84 2024-11-13T05:52:16,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/9d800bc5446a4800afc8ec71d4e6f42f 2024-11-13T05:52:16,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/96e6b46300b4420f978d2452874367be 2024-11-13T05:52:16,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/f56422f53f1748b49211d718f8be3d48 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/f56422f53f1748b49211d718f8be3d48 2024-11-13T05:52:16,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/0c553f7648064faf84ddb96eb06564d9 2024-11-13T05:52:16,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/5cb2b706e2014aa695c0e162e9414809 2024-11-13T05:52:16,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6e41847900464fb3b4e6a32718b965eb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/52063ecdfaf345b09333fe8a18927755, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3e073246f37f446db6c918addb106f28, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4815e033edda4d1ca446e4578bdc731a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/71fbca0ac9434ff1b104e70a7210b775, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/5ea7514035ac40aa860fae73c6742941, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d99a255b70e24e2e8315faaa76db1d11, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/454e00e9bb8a4aa6b32d3772f4081fe6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/cd6dd74614b04807af9061792c6be342, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/366bf23b5e354485aefee5b253999e68, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a674949a1114612a09a8f8ffaecff9a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/1c32e110dbf44e4c9f63fc13837bb8a8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/db1bd15af86f4456b9e651ebac893765, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d2323a6984e94646b980f779c2092c67, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/dd69e7c5061c445e8b0f5bb8394b08c7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a45433149d584c8baedd0c4855db7f85, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9b38bd93a9b140c496cd2756cdbc7bb7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/89a778ffb6ea4cffbb1505532460d15b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a895683683a4ee2ba5bc824b3330028, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ba5acbcf1e7145f4b2af90d427f2ff78, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ea956b2f17c144a1a8b3b7ed2c5e60ea, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4d6335d6477b4977ab56684c65f6dd7b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/29765679f2cf49e49a8bf2f8c822f86c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a92d68ca8bc8439e803ca4c1adf337f1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/b6d936f4696d4df7893d163e82fba8a2, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/c3441010303d491f962605cd6f07aa20] to archive 2024-11-13T05:52:16,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:52:16,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6e41847900464fb3b4e6a32718b965eb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6e41847900464fb3b4e6a32718b965eb 2024-11-13T05:52:16,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/52063ecdfaf345b09333fe8a18927755 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/52063ecdfaf345b09333fe8a18927755 2024-11-13T05:52:16,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3e073246f37f446db6c918addb106f28 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3e073246f37f446db6c918addb106f28 2024-11-13T05:52:16,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4815e033edda4d1ca446e4578bdc731a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4815e033edda4d1ca446e4578bdc731a 2024-11-13T05:52:16,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/71fbca0ac9434ff1b104e70a7210b775 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/71fbca0ac9434ff1b104e70a7210b775 2024-11-13T05:52:16,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/5ea7514035ac40aa860fae73c6742941 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/5ea7514035ac40aa860fae73c6742941 2024-11-13T05:52:16,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d99a255b70e24e2e8315faaa76db1d11 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d99a255b70e24e2e8315faaa76db1d11 2024-11-13T05:52:16,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/454e00e9bb8a4aa6b32d3772f4081fe6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/454e00e9bb8a4aa6b32d3772f4081fe6 2024-11-13T05:52:16,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/cd6dd74614b04807af9061792c6be342 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/cd6dd74614b04807af9061792c6be342 2024-11-13T05:52:16,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/366bf23b5e354485aefee5b253999e68 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/366bf23b5e354485aefee5b253999e68 2024-11-13T05:52:16,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a674949a1114612a09a8f8ffaecff9a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a674949a1114612a09a8f8ffaecff9a 2024-11-13T05:52:16,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/1c32e110dbf44e4c9f63fc13837bb8a8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/1c32e110dbf44e4c9f63fc13837bb8a8 2024-11-13T05:52:16,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/db1bd15af86f4456b9e651ebac893765 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/db1bd15af86f4456b9e651ebac893765 2024-11-13T05:52:16,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d2323a6984e94646b980f779c2092c67 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/d2323a6984e94646b980f779c2092c67 2024-11-13T05:52:16,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/dd69e7c5061c445e8b0f5bb8394b08c7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/dd69e7c5061c445e8b0f5bb8394b08c7 2024-11-13T05:52:16,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a45433149d584c8baedd0c4855db7f85 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a45433149d584c8baedd0c4855db7f85 2024-11-13T05:52:16,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9b38bd93a9b140c496cd2756cdbc7bb7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9b38bd93a9b140c496cd2756cdbc7bb7 2024-11-13T05:52:16,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/89a778ffb6ea4cffbb1505532460d15b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/89a778ffb6ea4cffbb1505532460d15b 2024-11-13T05:52:16,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a895683683a4ee2ba5bc824b3330028 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/9a895683683a4ee2ba5bc824b3330028 2024-11-13T05:52:16,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ba5acbcf1e7145f4b2af90d427f2ff78 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ba5acbcf1e7145f4b2af90d427f2ff78 2024-11-13T05:52:16,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ea956b2f17c144a1a8b3b7ed2c5e60ea to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/ea956b2f17c144a1a8b3b7ed2c5e60ea 2024-11-13T05:52:16,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4d6335d6477b4977ab56684c65f6dd7b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/4d6335d6477b4977ab56684c65f6dd7b 2024-11-13T05:52:16,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/29765679f2cf49e49a8bf2f8c822f86c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/29765679f2cf49e49a8bf2f8c822f86c 2024-11-13T05:52:16,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a92d68ca8bc8439e803ca4c1adf337f1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/a92d68ca8bc8439e803ca4c1adf337f1 2024-11-13T05:52:16,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/b6d936f4696d4df7893d163e82fba8a2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/b6d936f4696d4df7893d163e82fba8a2 2024-11-13T05:52:16,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/c3441010303d491f962605cd6f07aa20 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/c3441010303d491f962605cd6f07aa20 2024-11-13T05:52:16,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2d21f2fbc05441e881f7da326f42d18b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/6e5bf9889c9644fa99e1e58400868b50, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/99ac8ab98b8342efb3a80a74522d017e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e6f9a22018c54036862b95ef2a902988, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/983a22407bbe4089a2a625ea2d6322ca, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a08f121a86104282a8342eeefb0c1bbb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a97aa9dc8b234177baef7d16e2c9857c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ab0c0711212148219120a689b4c75567, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5b448dec1cbb49f889169c39a0a71409, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f007a36f7afe47d1a01518e141d2ed5c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/65fedb526d8f4aa0bd9a9da52938bb63, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/808482f01cdb44b28e2176f7ace9f115, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/9d5e35d49b5a47e88b4f759c810d479b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ffe2c8c8bd064e55878c337ce13220ba, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4895169c3e2049359f351e72a38434cc, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f9c0992b22f04089b7b5c6e2ea611b00, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/36ae9e5f443240ccbced3985729e800d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4938153051904254ba41691c8b34db73, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/8682e3587dea4b2b8c1a9040582aae9c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/20500e481da44a0689fd097c8b345937, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5272c8638ee64070b89bb21161be1d5b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a1a297b807534368bf79935e188ac80b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/06b04ec2ab044d72b514c8db99f43eab, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/3cc7343461d641479a69c35bbf2f3326, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2c02846becb140019e9e5641a1540b9b] to archive 2024-11-13T05:52:16,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:52:16,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2d21f2fbc05441e881f7da326f42d18b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2d21f2fbc05441e881f7da326f42d18b 2024-11-13T05:52:16,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/6e5bf9889c9644fa99e1e58400868b50 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/6e5bf9889c9644fa99e1e58400868b50 2024-11-13T05:52:16,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/99ac8ab98b8342efb3a80a74522d017e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/99ac8ab98b8342efb3a80a74522d017e 2024-11-13T05:52:16,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e6f9a22018c54036862b95ef2a902988 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e6f9a22018c54036862b95ef2a902988 2024-11-13T05:52:16,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a6eb7f0609ae4a45ad3e1f7f4bdc7986 2024-11-13T05:52:16,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/983a22407bbe4089a2a625ea2d6322ca to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/983a22407bbe4089a2a625ea2d6322ca 2024-11-13T05:52:16,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a08f121a86104282a8342eeefb0c1bbb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a08f121a86104282a8342eeefb0c1bbb 2024-11-13T05:52:16,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a97aa9dc8b234177baef7d16e2c9857c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a97aa9dc8b234177baef7d16e2c9857c 2024-11-13T05:52:16,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ab0c0711212148219120a689b4c75567 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ab0c0711212148219120a689b4c75567 2024-11-13T05:52:16,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5b448dec1cbb49f889169c39a0a71409 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5b448dec1cbb49f889169c39a0a71409 2024-11-13T05:52:16,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f007a36f7afe47d1a01518e141d2ed5c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f007a36f7afe47d1a01518e141d2ed5c 2024-11-13T05:52:16,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/65fedb526d8f4aa0bd9a9da52938bb63 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/65fedb526d8f4aa0bd9a9da52938bb63 2024-11-13T05:52:16,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/808482f01cdb44b28e2176f7ace9f115 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/808482f01cdb44b28e2176f7ace9f115 2024-11-13T05:52:16,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/9d5e35d49b5a47e88b4f759c810d479b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/9d5e35d49b5a47e88b4f759c810d479b 2024-11-13T05:52:16,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ffe2c8c8bd064e55878c337ce13220ba to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/ffe2c8c8bd064e55878c337ce13220ba 2024-11-13T05:52:16,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4895169c3e2049359f351e72a38434cc to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4895169c3e2049359f351e72a38434cc 2024-11-13T05:52:16,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f9c0992b22f04089b7b5c6e2ea611b00 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/f9c0992b22f04089b7b5c6e2ea611b00 2024-11-13T05:52:16,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/36ae9e5f443240ccbced3985729e800d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/36ae9e5f443240ccbced3985729e800d 2024-11-13T05:52:16,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4938153051904254ba41691c8b34db73 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/4938153051904254ba41691c8b34db73 2024-11-13T05:52:16,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/8682e3587dea4b2b8c1a9040582aae9c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/8682e3587dea4b2b8c1a9040582aae9c 2024-11-13T05:52:16,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/20500e481da44a0689fd097c8b345937 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/20500e481da44a0689fd097c8b345937 2024-11-13T05:52:16,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5272c8638ee64070b89bb21161be1d5b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/5272c8638ee64070b89bb21161be1d5b 2024-11-13T05:52:16,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a1a297b807534368bf79935e188ac80b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/a1a297b807534368bf79935e188ac80b 2024-11-13T05:52:16,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/06b04ec2ab044d72b514c8db99f43eab to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/06b04ec2ab044d72b514c8db99f43eab 2024-11-13T05:52:16,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/3cc7343461d641479a69c35bbf2f3326 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/3cc7343461d641479a69c35bbf2f3326 2024-11-13T05:52:16,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2c02846becb140019e9e5641a1540b9b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/2c02846becb140019e9e5641a1540b9b 2024-11-13T05:52:16,191 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/recovered.edits/356.seqid, newMaxSeqId=356, maxSeqId=4 2024-11-13T05:52:16,192 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. 2024-11-13T05:52:16,192 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 49578a99aa8fa6a60ec17a19ef73c0fd: Waiting for close lock at 1731477133057Running coprocessor pre-close hooks at 1731477133057Disabling compacts and flushes for region at 1731477133057Disabling writes for close at 1731477134447 (+1390 ms)Obtaining lock to block concurrent updates at 1731477134447Preparing flush snapshotting stores in 49578a99aa8fa6a60ec17a19ef73c0fd at 1731477134447Finished memstore snapshotting TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731477134447Flushing stores of TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd. at 1731477134447Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/A: creating writer at 1731477134447Flushing mob file 49578a99aa8fa6a60ec17a19ef73c0fd/A: appending metadata at 1731477134451 (+4 ms)Flushing mob file 49578a99aa8fa6a60ec17a19ef73c0fd/A: closing flushed file at 1731477134451Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/A: appending metadata at 1731477134860 (+409 ms)Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/A: closing flushed file at 1731477134860Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/B: creating writer at 1731477135268 (+408 ms)Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/B: appending metadata at 1731477135270 (+2 ms)Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/B: closing flushed file at 1731477135270Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/C: creating writer at 1731477135680 (+410 ms)Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/C: appending metadata at 1731477135683 (+3 ms)Flushing 49578a99aa8fa6a60ec17a19ef73c0fd/C: closing flushed file at 1731477135683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52ae6a6b: reopening flushed file at 1731477136097 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d7c7e08: reopening flushed file at 1731477136102 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5675312b: reopening flushed file at 1731477136105 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 49578a99aa8fa6a60ec17a19ef73c0fd in 1662ms, sequenceid=353, compaction requested=true at 1731477136109 (+4 ms)Writing region close event to WAL at 1731477136188 (+79 ms)Running coprocessor post-close hooks at 1731477136192 (+4 ms)Closed at 1731477136192 2024-11-13T05:52:16,194 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:16,194 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=49578a99aa8fa6a60ec17a19ef73c0fd, regionState=CLOSED 2024-11-13T05:52:16,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:52:16,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-13T05:52:16,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 49578a99aa8fa6a60ec17a19ef73c0fd, server=bc386b7bf2fe,38979,1731477016232 in 3.2920 sec 2024-11-13T05:52:16,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-13T05:52:16,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49578a99aa8fa6a60ec17a19ef73c0fd, UNASSIGN in 3.2970 sec 2024-11-13T05:52:16,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T05:52:16,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3020 sec 2024-11-13T05:52:16,206 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477136205"}]},"ts":"1731477136205"} 2024-11-13T05:52:16,207 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:52:16,207 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:52:16,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3130 sec 2024-11-13T05:52:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-13T05:52:17,052 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:17,052 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:52:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:52:17,055 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,057 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,059 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/recovered.edits] 2024-11-13T05:52:17,062 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/1f98d2a66b404039b64e398ee294b3de to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/1f98d2a66b404039b64e398ee294b3de 2024-11-13T05:52:17,063 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/89f4f6d639d4407ab9c8c7d7f8ad561a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/89f4f6d639d4407ab9c8c7d7f8ad561a 2024-11-13T05:52:17,065 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a4643897b44a4578b2309ce9cdfb337a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/A/a4643897b44a4578b2309ce9cdfb337a 2024-11-13T05:52:17,067 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3db51c9398574a9a9a2f7219f1e8bbb8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/3db51c9398574a9a9a2f7219f1e8bbb8 2024-11-13T05:52:17,068 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6f6d506e913746b5b39f212bd4e9337b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/6f6d506e913746b5b39f212bd4e9337b 2024-11-13T05:52:17,070 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/95fefdf83f604cedb9bbb5d1398ffb42 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/B/95fefdf83f604cedb9bbb5d1398ffb42 2024-11-13T05:52:17,072 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/184d96550ca04a8ab04c7b6046cbce6e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/184d96550ca04a8ab04c7b6046cbce6e 2024-11-13T05:52:17,074 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/b6c414e18ef243658d3ee3dac6c9f5c2 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/b6c414e18ef243658d3ee3dac6c9f5c2 2024-11-13T05:52:17,075 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e0d0394e4a014dbfb0c98e38b2c82203 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/C/e0d0394e4a014dbfb0c98e38b2c82203 2024-11-13T05:52:17,079 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/recovered.edits/356.seqid to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd/recovered.edits/356.seqid 2024-11-13T05:52:17,080 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,080 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:52:17,080 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:52:17,081 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T05:52:17,086 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113056910ad3c3b45689b5c087ae29b4ca8_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113056910ad3c3b45689b5c087ae29b4ca8_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,088 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111309747f91e37a4951b0979fbe948b02ef_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111309747f91e37a4951b0979fbe948b02ef_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,090 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132173226fed704edf95d993e3eec2b60a_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132173226fed704edf95d993e3eec2b60a_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,091 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111326c44876bc764a0fb0333a4b5c80b853_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111326c44876bc764a0fb0333a4b5c80b853_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,093 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f6d6bf1d0f34dff8e2b908ea9264760_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133f6d6bf1d0f34dff8e2b908ea9264760_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,095 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347794978d8604089b460af310fe5f691_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347794978d8604089b460af310fe5f691_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,096 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376761befd9114d3496903cd16d0fc383_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376761befd9114d3496903cd16d0fc383_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,098 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113771e186e262449c6a947af66b74d7759_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113771e186e262449c6a947af66b74d7759_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,100 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138a649ad530bd434cb59974526753a5c7_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138a649ad530bd434cb59974526753a5c7_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,101 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113948b07ab28d64d9eba924085144ea80a_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113948b07ab28d64d9eba924085144ea80a_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,102 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c50a1c634cc4ec287113ff98e12c54d_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411139c50a1c634cc4ec287113ff98e12c54d_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,104 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa28eb9c75864d979f7cd3c8bc3b9e02_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa28eb9c75864d979f7cd3c8bc3b9e02_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,106 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1bd823c7a524b3186c97c26e0570f96_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b1bd823c7a524b3186c97c26e0570f96_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,108 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bb8fef864cb84603882ea5ec8223357c_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bb8fef864cb84603882ea5ec8223357c_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,110 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cde3b646043240a69c1757ab200d464e_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cde3b646043240a69c1757ab200d464e_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,112 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d7bce21e91c04cb3a9119314add0bc35_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d7bce21e91c04cb3a9119314add0bc35_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,113 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113db90b59e3d654c3b9dd8bea04c05d623_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113db90b59e3d654c3b9dd8bea04c05d623_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,115 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e4a9b15a24bf4c64998ec2fd9c09526b_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e4a9b15a24bf4c64998ec2fd9c09526b_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,117 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8714fd45f394c0c80d9148bac26f61d_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8714fd45f394c0c80d9148bac26f61d_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,120 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fccfe764ea594d58803054c8ac70e20a_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fccfe764ea594d58803054c8ac70e20a_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,122 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fe0ba38b7170468993e5f0514058e756_49578a99aa8fa6a60ec17a19ef73c0fd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fe0ba38b7170468993e5f0514058e756_49578a99aa8fa6a60ec17a19ef73c0fd 2024-11-13T05:52:17,123 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:52:17,127 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,129 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:52:17,132 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:52:17,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,133 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:52:17,134 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731477137133"}]},"ts":"9223372036854775807"} 2024-11-13T05:52:17,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:52:17,136 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 49578a99aa8fa6a60ec17a19ef73c0fd, NAME => 'TestAcidGuarantees,,1731477106136.49578a99aa8fa6a60ec17a19ef73c0fd.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:52:17,136 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:52:17,136 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731477137136"}]},"ts":"9223372036854775807"} 2024-11-13T05:52:17,138 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:52:17,139 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-13T05:52:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-13T05:52:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:52:17,162 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:17,171 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 250), OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=7294 (was 6891) - AvailableMemoryMB LEAK? - 2024-11-13T05:52:17,181 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=7294 2024-11-13T05:52:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:52:17,182 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:17,185 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:52:17,185 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:17,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-13T05:52:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T05:52:17,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:52:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742282_1458 (size=1061) 2024-11-13T05:52:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T05:52:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T05:52:17,594 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:52:17,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742283_1459 (size=53) 2024-11-13T05:52:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T05:52:18,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:18,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d5b6967321153590da5042dc4609c1e3, disabling compactions & flushes 2024-11-13T05:52:18,001 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. after waiting 0 ms 2024-11-13T05:52:18,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,001 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d5b6967321153590da5042dc4609c1e3: Waiting for close lock at 1731477138001Disabling compacts and flushes for region at 1731477138001Disabling writes for close at 1731477138001Writing region close event to WAL at 1731477138001Closed at 1731477138001 2024-11-13T05:52:18,003 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:52:18,003 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731477138003"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477138003"}]},"ts":"1731477138003"} 2024-11-13T05:52:18,005 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:52:18,007 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:52:18,007 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477138007"}]},"ts":"1731477138007"} 2024-11-13T05:52:18,010 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:52:18,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, ASSIGN}] 2024-11-13T05:52:18,012 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, ASSIGN 2024-11-13T05:52:18,012 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=false 2024-11-13T05:52:18,163 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=d5b6967321153590da5042dc4609c1e3, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:18,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, ASSIGN because future has completed 2024-11-13T05:52:18,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:52:18,319 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,319 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:18,320 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,320 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:18,320 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,320 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,321 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T05:52:18,322 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:18,322 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5b6967321153590da5042dc4609c1e3 columnFamilyName A 2024-11-13T05:52:18,322 DEBUG [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:18,323 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.HStore(327): Store=d5b6967321153590da5042dc4609c1e3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:18,323 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,324 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:18,324 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5b6967321153590da5042dc4609c1e3 columnFamilyName B 2024-11-13T05:52:18,324 DEBUG [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:18,324 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.HStore(327): Store=d5b6967321153590da5042dc4609c1e3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:18,324 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,325 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:18,325 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5b6967321153590da5042dc4609c1e3 columnFamilyName C 2024-11-13T05:52:18,325 DEBUG [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:18,325 INFO [StoreOpener-d5b6967321153590da5042dc4609c1e3-1 {}] regionserver.HStore(327): Store=d5b6967321153590da5042dc4609c1e3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:18,326 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,326 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,326 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,327 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,327 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,327 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,328 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:52:18,328 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,330 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:18,330 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened d5b6967321153590da5042dc4609c1e3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61606825, jitterRate=-0.08198677003383636}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:52:18,330 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:18,331 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for d5b6967321153590da5042dc4609c1e3: Running coprocessor pre-open hook at 1731477138320Writing region info on filesystem at 1731477138320Initializing all the Stores at 1731477138321 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477138321Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477138321Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477138321Cleaning up temporary data from old regions at 1731477138327 (+6 ms)Running coprocessor post-open hooks at 1731477138331 (+4 ms)Region opened successfully at 1731477138331 2024-11-13T05:52:18,332 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., pid=120, masterSystemTime=1731477138316 2024-11-13T05:52:18,334 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,334 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:18,334 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=d5b6967321153590da5042dc4609c1e3, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:18,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:52:18,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T05:52:18,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 in 171 msec 2024-11-13T05:52:18,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-13T05:52:18,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, ASSIGN in 328 msec 2024-11-13T05:52:18,341 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:52:18,341 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477138341"}]},"ts":"1731477138341"} 2024-11-13T05:52:18,342 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:52:18,343 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:52:18,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-13T05:52:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-13T05:52:19,332 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:19,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4499c466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,334 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c9965f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd29660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,336 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,339 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:52:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:52:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56b14b02 2024-11-13T05:52:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:52:19,341 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:52:19,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ff7534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,343 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4a4f24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,344 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4714c793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6cc063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e94039a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6346fce4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf63333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,349 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427a6b16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,350 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1686e03c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fd006e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,352 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bab3e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62be31af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fddce7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,357 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698f943b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74921574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541036fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,362 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5178ed76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,363 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eb2b15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c0d5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,365 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff4c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,367 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@770428c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4025740, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,369 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45120b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,370 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbf9bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca86de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,376 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5375345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,377 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72bce3c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:19,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:19,379 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc64dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:19,380 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45761629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:19,381 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:19,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T05:52:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:52:19,383 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:19,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,384 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:19,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:19,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,391 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,394 DEBUG [Thread-2051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,394 DEBUG [Thread-2066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,395 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,396 DEBUG [Thread-2070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,397 DEBUG [Thread-2062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,399 DEBUG [Thread-2068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,402 DEBUG [Thread-2064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,403 DEBUG [Thread-2064 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,403 DEBUG [Thread-2064 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,403 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:19,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,404 DEBUG [Thread-2059 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,406 DEBUG [Thread-2059 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:19,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:52:19,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:19,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:19,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:19,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2] 2024-11-13T05:52:19,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5b2c9b6325be42478a2331b395d827be is 50, key is test_row_0/A:col10/1731477139403/Put/seqid=0 2024-11-13T05:52:19,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477199437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477199439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477199441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477199451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477199452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:52:19,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742284_1460 (size=12001) 2024-11-13T05:52:19,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5b2c9b6325be42478a2331b395d827be 2024-11-13T05:52:19,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:52:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:19,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:19,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/39faa31eb07c4e4d80927f342cc2ebe5 is 50, key is test_row_0/B:col10/1731477139403/Put/seqid=0 2024-11-13T05:52:19,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742285_1461 (size=12001) 2024-11-13T05:52:19,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/39faa31eb07c4e4d80927f342cc2ebe5 2024-11-13T05:52:19,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477199562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477199562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477199562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477199562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477199572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/ddd18ea56183435fb00f1d77cba17b9a is 50, key is test_row_0/C:col10/1731477139403/Put/seqid=0 2024-11-13T05:52:19,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742286_1462 (size=12001) 2024-11-13T05:52:19,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/ddd18ea56183435fb00f1d77cba17b9a 2024-11-13T05:52:19,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5b2c9b6325be42478a2331b395d827be as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5b2c9b6325be42478a2331b395d827be 2024-11-13T05:52:19,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5b2c9b6325be42478a2331b395d827be, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:52:19,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/39faa31eb07c4e4d80927f342cc2ebe5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/39faa31eb07c4e4d80927f342cc2ebe5 2024-11-13T05:52:19,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/39faa31eb07c4e4d80927f342cc2ebe5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:52:19,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/ddd18ea56183435fb00f1d77cba17b9a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ddd18ea56183435fb00f1d77cba17b9a 2024-11-13T05:52:19,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ddd18ea56183435fb00f1d77cba17b9a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:52:19,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d5b6967321153590da5042dc4609c1e3 in 226ms, sequenceid=14, compaction requested=false 2024-11-13T05:52:19,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:19,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T05:52:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:19,692 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T05:52:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:52:19,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/73c5b40a4f5f45dd9e53992796fadc32 is 50, key is test_row_0/A:col10/1731477139434/Put/seqid=0 2024-11-13T05:52:19,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742287_1463 (size=12001) 2024-11-13T05:52:19,736 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/73c5b40a4f5f45dd9e53992796fadc32 2024-11-13T05:52:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/ef1f9f673ce948a28b4cc90f3c6437b5 is 50, key is test_row_0/B:col10/1731477139434/Put/seqid=0 2024-11-13T05:52:19,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742288_1464 (size=12001) 2024-11-13T05:52:19,769 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/ef1f9f673ce948a28b4cc90f3c6437b5 2024-11-13T05:52:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:19,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477199792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477199798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477199803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5313a69a250649daa7d77793c1cab218 is 50, key is test_row_0/C:col10/1731477139434/Put/seqid=0 2024-11-13T05:52:19,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477199806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477199804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742289_1465 (size=12001) 2024-11-13T05:52:19,861 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5313a69a250649daa7d77793c1cab218 2024-11-13T05:52:19,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/73c5b40a4f5f45dd9e53992796fadc32 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/73c5b40a4f5f45dd9e53992796fadc32 2024-11-13T05:52:19,874 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/73c5b40a4f5f45dd9e53992796fadc32, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T05:52:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/ef1f9f673ce948a28b4cc90f3c6437b5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/ef1f9f673ce948a28b4cc90f3c6437b5 2024-11-13T05:52:19,879 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/ef1f9f673ce948a28b4cc90f3c6437b5, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T05:52:19,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5313a69a250649daa7d77793c1cab218 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5313a69a250649daa7d77793c1cab218 2024-11-13T05:52:19,888 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5313a69a250649daa7d77793c1cab218, entries=150, sequenceid=33, filesize=11.7 K 2024-11-13T05:52:19,889 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d5b6967321153590da5042dc4609c1e3 in 197ms, sequenceid=33, compaction requested=false 2024-11-13T05:52:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T05:52:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T05:52:19,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T05:52:19,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 507 msec 2024-11-13T05:52:19,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 519 msec 2024-11-13T05:52:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:19,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:52:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:19,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:19,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/776c1e02af844462ab0bc84b37c15948 is 50, key is test_row_0/A:col10/1731477139924/Put/seqid=0 2024-11-13T05:52:19,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742290_1466 (size=14341) 2024-11-13T05:52:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477199972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477199973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477199974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477199976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T05:52:20,011 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:20,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T05:52:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:20,014 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:20,015 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:20,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477200101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477200101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477200101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477200101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477200112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:20,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T05:52:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T05:52:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477200322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477200322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477200322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477200323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/776c1e02af844462ab0bc84b37c15948 2024-11-13T05:52:20,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/94ea2fad41aa4d63b4d105cafa09901a is 50, key is test_row_0/B:col10/1731477139924/Put/seqid=0 2024-11-13T05:52:20,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742291_1467 (size=12001) 2024-11-13T05:52:20,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/94ea2fad41aa4d63b4d105cafa09901a 2024-11-13T05:52:20,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/2de92af2a13a42beafc6098827a8ae02 is 50, key is test_row_0/C:col10/1731477139924/Put/seqid=0 2024-11-13T05:52:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742292_1468 (size=12001) 2024-11-13T05:52:20,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T05:52:20,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:20,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477200631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T05:52:20,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:20,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477200642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477200643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477200645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477200645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:20,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T05:52:20,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:20,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/2de92af2a13a42beafc6098827a8ae02 2024-11-13T05:52:20,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/776c1e02af844462ab0bc84b37c15948 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/776c1e02af844462ab0bc84b37c15948 2024-11-13T05:52:20,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/776c1e02af844462ab0bc84b37c15948, entries=200, sequenceid=47, filesize=14.0 K 2024-11-13T05:52:20,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/94ea2fad41aa4d63b4d105cafa09901a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/94ea2fad41aa4d63b4d105cafa09901a 2024-11-13T05:52:20,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/94ea2fad41aa4d63b4d105cafa09901a, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T05:52:20,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/2de92af2a13a42beafc6098827a8ae02 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2de92af2a13a42beafc6098827a8ae02 2024-11-13T05:52:20,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2de92af2a13a42beafc6098827a8ae02, entries=150, sequenceid=47, filesize=11.7 K 2024-11-13T05:52:20,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d5b6967321153590da5042dc4609c1e3 in 976ms, sequenceid=47, compaction requested=true 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:20,901 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:20,901 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:20,902 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:20,902 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:20,902 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:20,902 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:20,902 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,902 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,902 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5b2c9b6325be42478a2331b395d827be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/73c5b40a4f5f45dd9e53992796fadc32, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/776c1e02af844462ab0bc84b37c15948] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=37.4 K 2024-11-13T05:52:20,902 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/39faa31eb07c4e4d80927f342cc2ebe5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/ef1f9f673ce948a28b4cc90f3c6437b5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/94ea2fad41aa4d63b4d105cafa09901a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=35.2 K 2024-11-13T05:52:20,903 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b2c9b6325be42478a2331b395d827be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477139403 2024-11-13T05:52:20,903 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 39faa31eb07c4e4d80927f342cc2ebe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477139403 2024-11-13T05:52:20,903 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c5b40a4f5f45dd9e53992796fadc32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731477139434 2024-11-13T05:52:20,903 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1f9f673ce948a28b4cc90f3c6437b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731477139434 2024-11-13T05:52:20,903 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776c1e02af844462ab0bc84b37c15948, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477139780 2024-11-13T05:52:20,903 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ea2fad41aa4d63b4d105cafa09901a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477139802 2024-11-13T05:52:20,929 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:20,929 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e8b3d68e57b64005bb79c3c1ed05c568 is 50, key is test_row_0/B:col10/1731477139924/Put/seqid=0 2024-11-13T05:52:20,944 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:20,945 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/e457233686ab4a788fa3729ce61fd47b is 50, key is test_row_0/A:col10/1731477139924/Put/seqid=0 2024-11-13T05:52:20,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T05:52:20,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,948 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-13T05:52:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742293_1469 (size=12104) 2024-11-13T05:52:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/675553802af14732b3426b85fdc649d8 is 50, key is test_row_0/A:col10/1731477139974/Put/seqid=0 2024-11-13T05:52:20,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742294_1470 (size=12104) 2024-11-13T05:52:20,968 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/e457233686ab4a788fa3729ce61fd47b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e457233686ab4a788fa3729ce61fd47b 2024-11-13T05:52:20,973 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into e457233686ab4a788fa3729ce61fd47b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:20,973 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:20,973 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=13, startTime=1731477140901; duration=0sec 2024-11-13T05:52:20,973 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:20,974 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:20,974 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:20,975 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:20,976 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:20,976 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:20,976 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ddd18ea56183435fb00f1d77cba17b9a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5313a69a250649daa7d77793c1cab218, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2de92af2a13a42beafc6098827a8ae02] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=35.2 K 2024-11-13T05:52:20,976 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddd18ea56183435fb00f1d77cba17b9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477139403 2024-11-13T05:52:20,977 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5313a69a250649daa7d77793c1cab218, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731477139434 2024-11-13T05:52:20,978 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de92af2a13a42beafc6098827a8ae02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477139802 2024-11-13T05:52:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742295_1471 (size=12001) 2024-11-13T05:52:20,985 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:20,986 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/2e83c31a51834686bd49a308b97a31db is 50, key is test_row_0/C:col10/1731477139924/Put/seqid=0 2024-11-13T05:52:20,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742296_1472 (size=12104) 2024-11-13T05:52:21,007 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/2e83c31a51834686bd49a308b97a31db as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2e83c31a51834686bd49a308b97a31db 2024-11-13T05:52:21,024 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 2e83c31a51834686bd49a308b97a31db(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:21,024 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:21,024 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=13, startTime=1731477140901; duration=0sec 2024-11-13T05:52:21,025 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:21,025 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:21,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477201186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477201187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477201188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477201201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477201312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477201312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477201312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477201321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,353 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e8b3d68e57b64005bb79c3c1ed05c568 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e8b3d68e57b64005bb79c3c1ed05c568 2024-11-13T05:52:21,358 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into e8b3d68e57b64005bb79c3c1ed05c568(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:21,358 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:21,358 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=13, startTime=1731477140901; duration=0sec 2024-11-13T05:52:21,358 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:21,358 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:21,380 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/675553802af14732b3426b85fdc649d8 2024-11-13T05:52:21,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/573c22297ab64eaf8750c8e7cc61a189 is 50, key is test_row_0/B:col10/1731477139974/Put/seqid=0 2024-11-13T05:52:21,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742297_1473 (size=12001) 2024-11-13T05:52:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477201522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477201531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477201531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477201541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477201641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,646 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,717 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:52:21,797 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/573c22297ab64eaf8750c8e7cc61a189 2024-11-13T05:52:21,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/9ad81f0223d74f5696add122336f0c2a is 50, key is test_row_0/C:col10/1731477139974/Put/seqid=0 2024-11-13T05:52:21,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742298_1474 (size=12001) 2024-11-13T05:52:21,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477201841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477201841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477201851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:21,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477201862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:22,208 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/9ad81f0223d74f5696add122336f0c2a 2024-11-13T05:52:22,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/675553802af14732b3426b85fdc649d8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/675553802af14732b3426b85fdc649d8 2024-11-13T05:52:22,215 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/675553802af14732b3426b85fdc649d8, entries=150, sequenceid=65, filesize=11.7 K 2024-11-13T05:52:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/573c22297ab64eaf8750c8e7cc61a189 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/573c22297ab64eaf8750c8e7cc61a189 2024-11-13T05:52:22,219 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/573c22297ab64eaf8750c8e7cc61a189, entries=150, sequenceid=65, filesize=11.7 K 2024-11-13T05:52:22,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/9ad81f0223d74f5696add122336f0c2a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/9ad81f0223d74f5696add122336f0c2a 2024-11-13T05:52:22,223 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/9ad81f0223d74f5696add122336f0c2a, entries=150, sequenceid=65, filesize=11.7 K 2024-11-13T05:52:22,224 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d5b6967321153590da5042dc4609c1e3 in 1275ms, sequenceid=65, compaction requested=false 2024-11-13T05:52:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T05:52:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T05:52:22,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T05:52:22,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2100 sec 2024-11-13T05:52:22,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2140 sec 2024-11-13T05:52:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:22,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-13T05:52:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:22,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/17dd4232b6a34a7dbf6703d6c5622d1e is 50, key is test_row_0/A:col10/1731477142367/Put/seqid=0 2024-11-13T05:52:22,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742299_1475 (size=16681) 2024-11-13T05:52:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477202391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477202393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477202394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477202398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477202501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477202511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477202512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477202721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477202731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477202732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477202732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:22,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/17dd4232b6a34a7dbf6703d6c5622d1e 2024-11-13T05:52:22,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0f052fdbc48a4f988884fe50cee9533c is 50, key is test_row_0/B:col10/1731477142367/Put/seqid=0 2024-11-13T05:52:22,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742300_1476 (size=12001) 2024-11-13T05:52:23,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477203041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477203041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477203041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477203041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0f052fdbc48a4f988884fe50cee9533c 2024-11-13T05:52:23,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d329fc057d544c7f879fd3bfdf75a075 is 50, key is test_row_0/C:col10/1731477142367/Put/seqid=0 2024-11-13T05:52:23,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742301_1477 (size=12001) 2024-11-13T05:52:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477203561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d329fc057d544c7f879fd3bfdf75a075 2024-11-13T05:52:23,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/17dd4232b6a34a7dbf6703d6c5622d1e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/17dd4232b6a34a7dbf6703d6c5622d1e 2024-11-13T05:52:23,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/17dd4232b6a34a7dbf6703d6c5622d1e, entries=250, sequenceid=84, filesize=16.3 K 2024-11-13T05:52:23,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0f052fdbc48a4f988884fe50cee9533c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0f052fdbc48a4f988884fe50cee9533c 2024-11-13T05:52:23,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0f052fdbc48a4f988884fe50cee9533c, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T05:52:23,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d329fc057d544c7f879fd3bfdf75a075 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d329fc057d544c7f879fd3bfdf75a075 2024-11-13T05:52:23,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d329fc057d544c7f879fd3bfdf75a075, entries=150, sequenceid=84, filesize=11.7 K 2024-11-13T05:52:23,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d5b6967321153590da5042dc4609c1e3 in 1242ms, sequenceid=84, compaction requested=true 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:23,610 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:23,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:52:23,611 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:23,612 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:23,612 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:23,612 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:23,612 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e457233686ab4a788fa3729ce61fd47b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/675553802af14732b3426b85fdc649d8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/17dd4232b6a34a7dbf6703d6c5622d1e] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=39.8 K 2024-11-13T05:52:23,612 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:23,612 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e457233686ab4a788fa3729ce61fd47b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477139802 2024-11-13T05:52:23,612 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:23,612 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:23,613 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e8b3d68e57b64005bb79c3c1ed05c568, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/573c22297ab64eaf8750c8e7cc61a189, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0f052fdbc48a4f988884fe50cee9533c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=35.3 K 2024-11-13T05:52:23,613 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 675553802af14732b3426b85fdc649d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731477139971 2024-11-13T05:52:23,613 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b3d68e57b64005bb79c3c1ed05c568, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477139802 2024-11-13T05:52:23,613 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17dd4232b6a34a7dbf6703d6c5622d1e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731477141181 2024-11-13T05:52:23,613 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 573c22297ab64eaf8750c8e7cc61a189, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731477139971 2024-11-13T05:52:23,613 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f052fdbc48a4f988884fe50cee9533c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731477141194 2024-11-13T05:52:23,622 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:23,622 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:23,622 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e6c160354c8e4329a5e018ff83c10427 is 50, key is test_row_0/B:col10/1731477142367/Put/seqid=0 2024-11-13T05:52:23,623 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/30cbfe54ce6341c289d00059d115be9f is 50, key is test_row_0/A:col10/1731477142367/Put/seqid=0 2024-11-13T05:52:23,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742302_1478 (size=12207) 2024-11-13T05:52:23,663 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e6c160354c8e4329a5e018ff83c10427 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e6c160354c8e4329a5e018ff83c10427 2024-11-13T05:52:23,667 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into e6c160354c8e4329a5e018ff83c10427(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:23,667 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:23,667 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=13, startTime=1731477143610; duration=0sec 2024-11-13T05:52:23,667 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:23,667 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:23,667 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:23,669 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:23,669 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:23,669 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:23,669 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2e83c31a51834686bd49a308b97a31db, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/9ad81f0223d74f5696add122336f0c2a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d329fc057d544c7f879fd3bfdf75a075] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=35.3 K 2024-11-13T05:52:23,669 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e83c31a51834686bd49a308b97a31db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731477139802 2024-11-13T05:52:23,669 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad81f0223d74f5696add122336f0c2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731477139971 2024-11-13T05:52:23,670 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d329fc057d544c7f879fd3bfdf75a075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731477141194 2024-11-13T05:52:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:23,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-13T05:52:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:23,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:23,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d3c0a3e06f95481b81f75d4901dec6d3 is 50, key is test_row_0/A:col10/1731477142392/Put/seqid=0 2024-11-13T05:52:23,677 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:23,678 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/3059347c63594231bb3f8723832f8a3a is 50, key is test_row_0/C:col10/1731477142367/Put/seqid=0 2024-11-13T05:52:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742303_1479 (size=12207) 2024-11-13T05:52:23,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742305_1481 (size=12207) 2024-11-13T05:52:23,690 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/30cbfe54ce6341c289d00059d115be9f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/30cbfe54ce6341c289d00059d115be9f 2024-11-13T05:52:23,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742304_1480 (size=14341) 2024-11-13T05:52:23,697 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into 30cbfe54ce6341c289d00059d115be9f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:23,697 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:23,697 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=13, startTime=1731477143610; duration=0sec 2024-11-13T05:52:23,697 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:23,697 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477203748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477203861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477204071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,091 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/3059347c63594231bb3f8723832f8a3a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/3059347c63594231bb3f8723832f8a3a 2024-11-13T05:52:24,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d3c0a3e06f95481b81f75d4901dec6d3 2024-11-13T05:52:24,095 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 3059347c63594231bb3f8723832f8a3a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:24,095 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:24,095 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=13, startTime=1731477143610; duration=0sec 2024-11-13T05:52:24,095 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:24,095 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:24,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/c59dadf69e694e278761789d0320d9e1 is 50, key is test_row_0/B:col10/1731477142392/Put/seqid=0 2024-11-13T05:52:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742306_1482 (size=12001) 2024-11-13T05:52:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T05:52:24,172 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:24,172 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T05:52:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T05:52:24,174 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:24,175 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:24,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T05:52:24,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T05:52:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477204392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T05:52:24,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:24,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T05:52:24,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/c59dadf69e694e278761789d0320d9e1 2024-11-13T05:52:24,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/32e4528ce5484be396e3fd96588c20b9 is 50, key is test_row_0/C:col10/1731477142392/Put/seqid=0 2024-11-13T05:52:24,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742307_1483 (size=12001) 2024-11-13T05:52:24,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477204581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477204581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477204581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,585 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:24,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,585 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477204581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,586 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,586 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T05:52:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T05:52:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T05:52:24,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/32e4528ce5484be396e3fd96588c20b9 2024-11-13T05:52:24,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d3c0a3e06f95481b81f75d4901dec6d3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d3c0a3e06f95481b81f75d4901dec6d3 2024-11-13T05:52:24,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d3c0a3e06f95481b81f75d4901dec6d3, entries=200, sequenceid=102, filesize=14.0 K 2024-11-13T05:52:24,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/c59dadf69e694e278761789d0320d9e1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/c59dadf69e694e278761789d0320d9e1 2024-11-13T05:52:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477204911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:24,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/c59dadf69e694e278761789d0320d9e1, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T05:52:24,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/32e4528ce5484be396e3fd96588c20b9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/32e4528ce5484be396e3fd96588c20b9 2024-11-13T05:52:24,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/32e4528ce5484be396e3fd96588c20b9, entries=150, sequenceid=102, filesize=11.7 K 2024-11-13T05:52:24,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d5b6967321153590da5042dc4609c1e3 in 1253ms, sequenceid=102, compaction requested=false 2024-11-13T05:52:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:24,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T05:52:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:24,938 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-13T05:52:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:24,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:24,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/b703e39df7a74f598c2cb87d228df048 is 50, key is test_row_0/A:col10/1731477143746/Put/seqid=0 2024-11-13T05:52:24,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742308_1484 (size=12001) 2024-11-13T05:52:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T05:52:25,347 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/b703e39df7a74f598c2cb87d228df048 2024-11-13T05:52:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/1544199806ce4bf28960f853c464f243 is 50, key is test_row_0/B:col10/1731477143746/Put/seqid=0 2024-11-13T05:52:25,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742309_1485 (size=12001) 2024-11-13T05:52:25,356 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/1544199806ce4bf28960f853c464f243 2024-11-13T05:52:25,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/0360f7f6fe7a4bff8c6c2bd5f602127e is 50, key is test_row_0/C:col10/1731477143746/Put/seqid=0 2024-11-13T05:52:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742310_1486 (size=12001) 2024-11-13T05:52:25,369 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/0360f7f6fe7a4bff8c6c2bd5f602127e 2024-11-13T05:52:25,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/b703e39df7a74f598c2cb87d228df048 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/b703e39df7a74f598c2cb87d228df048 2024-11-13T05:52:25,375 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/b703e39df7a74f598c2cb87d228df048, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T05:52:25,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/1544199806ce4bf28960f853c464f243 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1544199806ce4bf28960f853c464f243 2024-11-13T05:52:25,380 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1544199806ce4bf28960f853c464f243, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T05:52:25,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/0360f7f6fe7a4bff8c6c2bd5f602127e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/0360f7f6fe7a4bff8c6c2bd5f602127e 2024-11-13T05:52:25,386 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/0360f7f6fe7a4bff8c6c2bd5f602127e, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T05:52:25,387 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for d5b6967321153590da5042dc4609c1e3 in 450ms, sequenceid=119, compaction requested=true 2024-11-13T05:52:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T05:52:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T05:52:25,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T05:52:25,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2130 sec 2024-11-13T05:52:25,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.2180 sec 2024-11-13T05:52:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:25,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:52:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:25,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:25,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/70c806cc2de64fd9a3d62fbfca4765d0 is 50, key is test_row_0/A:col10/1731477145951/Put/seqid=0 2024-11-13T05:52:25,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742311_1487 (size=16731) 2024-11-13T05:52:26,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477146047, max timestamp=1731477146071Num uniques 859; 2024-11-13T05:52:26,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477146047, max timestamp=1731477146071Num uniques 859; 2024-11-13T05:52:26,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477146047, max timestamp=1731477146071Num uniques 859; 2024-11-13T05:52:26,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:26,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:26,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:26,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477206226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T05:52:26,332 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:26,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T05:52:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:26,334 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:26,334 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:26,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:26,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477206331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/70c806cc2de64fd9a3d62fbfca4765d0 2024-11-13T05:52:26,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/30340638f8764cf2bea1315f8a7ad8d9 is 50, key is test_row_0/B:col10/1731477145951/Put/seqid=0 2024-11-13T05:52:26,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742312_1488 (size=12051) 2024-11-13T05:52:26,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/30340638f8764cf2bea1315f8a7ad8d9 2024-11-13T05:52:26,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/06e062cfb15c4ea0a54ce49018eb13ef is 50, key is test_row_0/C:col10/1731477145951/Put/seqid=0 2024-11-13T05:52:26,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742313_1489 (size=12051) 2024-11-13T05:52:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:26,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:26,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477206551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477206592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,600 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477206601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477206601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,612 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,613 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477206611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,619 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:26,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:26,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:26,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:26,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/06e062cfb15c4ea0a54ce49018eb13ef 2024-11-13T05:52:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/70c806cc2de64fd9a3d62fbfca4765d0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/70c806cc2de64fd9a3d62fbfca4765d0 2024-11-13T05:52:26,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:26,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:26,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/70c806cc2de64fd9a3d62fbfca4765d0, entries=250, sequenceid=129, filesize=16.3 K 2024-11-13T05:52:26,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/30340638f8764cf2bea1315f8a7ad8d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/30340638f8764cf2bea1315f8a7ad8d9 2024-11-13T05:52:26,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/30340638f8764cf2bea1315f8a7ad8d9, entries=150, sequenceid=129, filesize=11.8 K 2024-11-13T05:52:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/06e062cfb15c4ea0a54ce49018eb13ef as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/06e062cfb15c4ea0a54ce49018eb13ef 2024-11-13T05:52:26,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/06e062cfb15c4ea0a54ce49018eb13ef, entries=150, sequenceid=129, filesize=11.8 K 2024-11-13T05:52:26,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d5b6967321153590da5042dc4609c1e3 in 863ms, sequenceid=129, compaction requested=true 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:26,820 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T05:52:26,820 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:26,822 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:26,822 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:26,822 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,822 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e6c160354c8e4329a5e018ff83c10427, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/c59dadf69e694e278761789d0320d9e1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1544199806ce4bf28960f853c464f243, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/30340638f8764cf2bea1315f8a7ad8d9] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=47.1 K 2024-11-13T05:52:26,822 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:26,822 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:26,822 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,822 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/30cbfe54ce6341c289d00059d115be9f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d3c0a3e06f95481b81f75d4901dec6d3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/b703e39df7a74f598c2cb87d228df048, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/70c806cc2de64fd9a3d62fbfca4765d0] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=54.0 K 2024-11-13T05:52:26,823 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30cbfe54ce6341c289d00059d115be9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731477141194 2024-11-13T05:52:26,823 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c160354c8e4329a5e018ff83c10427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731477141194 2024-11-13T05:52:26,823 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c0a3e06f95481b81f75d4901dec6d3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731477142389 2024-11-13T05:52:26,823 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting c59dadf69e694e278761789d0320d9e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731477142392 2024-11-13T05:52:26,823 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b703e39df7a74f598c2cb87d228df048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731477143712 2024-11-13T05:52:26,823 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1544199806ce4bf28960f853c464f243, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731477143712 2024-11-13T05:52:26,826 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c806cc2de64fd9a3d62fbfca4765d0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477145932 2024-11-13T05:52:26,826 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 30340638f8764cf2bea1315f8a7ad8d9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477145942 2024-11-13T05:52:26,846 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#408 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:26,846 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/98e03574900d447a90c2be868ff7db38 is 50, key is test_row_0/A:col10/1731477145951/Put/seqid=0 2024-11-13T05:52:26,850 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#409 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:26,850 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9c932ddf4f724e44bbc156489ce98fd1 is 50, key is test_row_0/B:col10/1731477145951/Put/seqid=0 2024-11-13T05:52:26,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-13T05:52:26,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:26,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:26,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:26,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:26,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:26,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:26,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742314_1490 (size=12393) 2024-11-13T05:52:26,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742315_1491 (size=12393) 2024-11-13T05:52:26,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/8ec0f34059644bc5867d49717e2bcee6 is 50, key is test_row_0/A:col10/1731477146213/Put/seqid=0 2024-11-13T05:52:26,904 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9c932ddf4f724e44bbc156489ce98fd1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9c932ddf4f724e44bbc156489ce98fd1 2024-11-13T05:52:26,905 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/98e03574900d447a90c2be868ff7db38 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/98e03574900d447a90c2be868ff7db38 2024-11-13T05:52:26,913 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into 98e03574900d447a90c2be868ff7db38(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:26,914 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=12, startTime=1731477146820; duration=0sec 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:26,913 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into 9c932ddf4f724e44bbc156489ce98fd1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:26,914 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=12, startTime=1731477146820; duration=0sec 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:26,914 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:26,917 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:26,917 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:26,917 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,917 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/3059347c63594231bb3f8723832f8a3a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/32e4528ce5484be396e3fd96588c20b9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/0360f7f6fe7a4bff8c6c2bd5f602127e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/06e062cfb15c4ea0a54ce49018eb13ef] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=47.1 K 2024-11-13T05:52:26,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742316_1492 (size=14541) 2024-11-13T05:52:26,922 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3059347c63594231bb3f8723832f8a3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731477141194 2024-11-13T05:52:26,922 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e4528ce5484be396e3fd96588c20b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731477142392 2024-11-13T05:52:26,923 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0360f7f6fe7a4bff8c6c2bd5f602127e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731477143712 2024-11-13T05:52:26,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/8ec0f34059644bc5867d49717e2bcee6 2024-11-13T05:52:26,934 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e062cfb15c4ea0a54ce49018eb13ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477145942 2024-11-13T05:52:26,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/d36d19c0b58d4a1ebc3cfc67e707b628 is 50, key is test_row_0/B:col10/1731477146213/Put/seqid=0 2024-11-13T05:52:26,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:26,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:26,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:26,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:26,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742317_1493 (size=12151) 2024-11-13T05:52:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:26,968 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:26,969 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b is 50, key is test_row_0/C:col10/1731477145951/Put/seqid=0 2024-11-13T05:52:26,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742318_1494 (size=12393) 2024-11-13T05:52:26,985 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b 2024-11-13T05:52:26,988 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 1b60ed3ec2c34cca9ba97d5faf4b2c4b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:26,988 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:26,988 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=12, startTime=1731477146820; duration=0sec 2024-11-13T05:52:26,988 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:26,988 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:26,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477206993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:27,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477207101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:27,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477207312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:27,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/d36d19c0b58d4a1ebc3cfc67e707b628 2024-11-13T05:52:27,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/4d7b839d1f8f445d8927b3b9ad2d4488 is 50, key is test_row_0/C:col10/1731477146213/Put/seqid=0 2024-11-13T05:52:27,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742319_1495 (size=12151) 2024-11-13T05:52:27,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:27,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:27,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:27,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477207621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:27,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:27,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/4d7b839d1f8f445d8927b3b9ad2d4488 2024-11-13T05:52:27,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/8ec0f34059644bc5867d49717e2bcee6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/8ec0f34059644bc5867d49717e2bcee6 2024-11-13T05:52:27,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/8ec0f34059644bc5867d49717e2bcee6, entries=200, sequenceid=167, filesize=14.2 K 2024-11-13T05:52:27,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/d36d19c0b58d4a1ebc3cfc67e707b628 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/d36d19c0b58d4a1ebc3cfc67e707b628 2024-11-13T05:52:27,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/d36d19c0b58d4a1ebc3cfc67e707b628, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T05:52:27,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/4d7b839d1f8f445d8927b3b9ad2d4488 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/4d7b839d1f8f445d8927b3b9ad2d4488 2024-11-13T05:52:27,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/4d7b839d1f8f445d8927b3b9ad2d4488, entries=150, sequenceid=167, filesize=11.9 K 2024-11-13T05:52:27,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for d5b6967321153590da5042dc4609c1e3 in 901ms, sequenceid=167, compaction requested=false 2024-11-13T05:52:27,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:27,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T05:52:27,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:27,864 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T05:52:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/cffea235d1b54c4188a184544bc877f4 is 50, key is test_row_0/A:col10/1731477146973/Put/seqid=0 2024-11-13T05:52:27,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742320_1496 (size=12151) 2024-11-13T05:52:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:28,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:28,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477148222, max timestamp=1731477148254Num uniques 859; 2024-11-13T05:52:28,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477148222, max timestamp=1731477148254Num uniques 859; 2024-11-13T05:52:28,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:28,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:28,275 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/cffea235d1b54c4188a184544bc877f4 2024-11-13T05:52:28,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/b7e225da9ee041e9901dae72f5113d6b is 50, key is test_row_0/B:col10/1731477146973/Put/seqid=0 2024-11-13T05:52:28,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742321_1497 (size=12151) 2024-11-13T05:52:28,284 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/b7e225da9ee041e9901dae72f5113d6b 2024-11-13T05:52:28,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/adb1f191cb4041fdb6e5469ea7955485 is 50, key is test_row_0/C:col10/1731477146973/Put/seqid=0 2024-11-13T05:52:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742322_1498 (size=12151) 2024-11-13T05:52:28,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477208374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:28,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:28,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477208491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:28,696 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/adb1f191cb4041fdb6e5469ea7955485 2024-11-13T05:52:28,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/cffea235d1b54c4188a184544bc877f4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/cffea235d1b54c4188a184544bc877f4 2024-11-13T05:52:28,702 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/cffea235d1b54c4188a184544bc877f4, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T05:52:28,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/b7e225da9ee041e9901dae72f5113d6b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/b7e225da9ee041e9901dae72f5113d6b 2024-11-13T05:52:28,705 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/b7e225da9ee041e9901dae72f5113d6b, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T05:52:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/adb1f191cb4041fdb6e5469ea7955485 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/adb1f191cb4041fdb6e5469ea7955485 2024-11-13T05:52:28,709 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/adb1f191cb4041fdb6e5469ea7955485, entries=150, sequenceid=179, filesize=11.9 K 2024-11-13T05:52:28,709 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for d5b6967321153590da5042dc4609c1e3 in 845ms, sequenceid=179, compaction requested=true 2024-11-13T05:52:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T05:52:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T05:52:28,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T05:52:28,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3770 sec 2024-11-13T05:52:28,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3810 sec 2024-11-13T05:52:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:28,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=145.36 KB heapSize=423.16 KB 2024-11-13T05:52:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:28,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5a6cf39e249a4457be34ab62b18cc137 is 50, key is test_row_0/A:col10/1731477148718/Put/seqid=0 2024-11-13T05:52:28,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742323_1499 (size=14541) 2024-11-13T05:52:28,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5a6cf39e249a4457be34ab62b18cc137 2024-11-13T05:52:28,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/2125c49307de4168a88bf893cb8805dd is 50, key is test_row_0/B:col10/1731477148718/Put/seqid=0 2024-11-13T05:52:28,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742324_1500 (size=12151) 2024-11-13T05:52:28,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/2125c49307de4168a88bf893cb8805dd 2024-11-13T05:52:28,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/37947eec86a0451ab8524bf2ae854f7b is 50, key is test_row_0/C:col10/1731477148718/Put/seqid=0 2024-11-13T05:52:28,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742325_1501 (size=12151) 2024-11-13T05:52:28,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477208765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:28,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:28,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477208881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477209102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:29,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/37947eec86a0451ab8524bf2ae854f7b 2024-11-13T05:52:29,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5a6cf39e249a4457be34ab62b18cc137 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a6cf39e249a4457be34ab62b18cc137 2024-11-13T05:52:29,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a6cf39e249a4457be34ab62b18cc137, entries=200, sequenceid=213, filesize=14.2 K 2024-11-13T05:52:29,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/2125c49307de4168a88bf893cb8805dd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/2125c49307de4168a88bf893cb8805dd 2024-11-13T05:52:29,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/2125c49307de4168a88bf893cb8805dd, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T05:52:29,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/37947eec86a0451ab8524bf2ae854f7b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/37947eec86a0451ab8524bf2ae854f7b 2024-11-13T05:52:29,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/37947eec86a0451ab8524bf2ae854f7b, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T05:52:29,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~423.11 KB/433264, currentSize=33.54 KB/34350 for d5b6967321153590da5042dc4609c1e3 in 457ms, sequenceid=213, compaction requested=true 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:29,176 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:29,176 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:29,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53626 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48846 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:29,177 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:29,177 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:29,177 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/98e03574900d447a90c2be868ff7db38, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/8ec0f34059644bc5867d49717e2bcee6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/cffea235d1b54c4188a184544bc877f4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a6cf39e249a4457be34ab62b18cc137] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=52.4 K 2024-11-13T05:52:29,177 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9c932ddf4f724e44bbc156489ce98fd1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/d36d19c0b58d4a1ebc3cfc67e707b628, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/b7e225da9ee041e9901dae72f5113d6b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/2125c49307de4168a88bf893cb8805dd] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=47.7 K 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e03574900d447a90c2be868ff7db38, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477145942 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c932ddf4f724e44bbc156489ce98fd1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477145942 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ec0f34059644bc5867d49717e2bcee6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731477146186 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d36d19c0b58d4a1ebc3cfc67e707b628, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731477146213 2024-11-13T05:52:29,177 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffea235d1b54c4188a184544bc877f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731477146913 2024-11-13T05:52:29,178 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e225da9ee041e9901dae72f5113d6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731477146913 2024-11-13T05:52:29,178 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a6cf39e249a4457be34ab62b18cc137, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731477148273 2024-11-13T05:52:29,178 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2125c49307de4168a88bf893cb8805dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731477148344 2024-11-13T05:52:29,185 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:29,185 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/546e7ad149454a3593ca5a7d7c82da16 is 50, key is test_row_0/B:col10/1731477148718/Put/seqid=0 2024-11-13T05:52:29,187 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:29,187 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/3cffbc7390534d688e697a683f79e1d9 is 50, key is test_row_0/A:col10/1731477148718/Put/seqid=0 2024-11-13T05:52:29,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742326_1502 (size=12629) 2024-11-13T05:52:29,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742327_1503 (size=12629) 2024-11-13T05:52:29,200 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/546e7ad149454a3593ca5a7d7c82da16 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/546e7ad149454a3593ca5a7d7c82da16 2024-11-13T05:52:29,203 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/3cffbc7390534d688e697a683f79e1d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/3cffbc7390534d688e697a683f79e1d9 2024-11-13T05:52:29,204 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into 546e7ad149454a3593ca5a7d7c82da16(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:29,205 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:29,205 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=12, startTime=1731477149176; duration=0sec 2024-11-13T05:52:29,205 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:29,205 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:29,205 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:29,209 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48846 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:29,209 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:29,209 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:29,209 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into 3cffbc7390534d688e697a683f79e1d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:29,209 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/4d7b839d1f8f445d8927b3b9ad2d4488, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/adb1f191cb4041fdb6e5469ea7955485, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/37947eec86a0451ab8524bf2ae854f7b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=47.7 K 2024-11-13T05:52:29,209 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:29,209 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=12, startTime=1731477149176; duration=0sec 2024-11-13T05:52:29,209 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:29,210 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:29,210 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b60ed3ec2c34cca9ba97d5faf4b2c4b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731477145942 2024-11-13T05:52:29,210 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d7b839d1f8f445d8927b3b9ad2d4488, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731477146213 2024-11-13T05:52:29,210 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting adb1f191cb4041fdb6e5469ea7955485, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731477146913 2024-11-13T05:52:29,210 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 37947eec86a0451ab8524bf2ae854f7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731477148344 2024-11-13T05:52:29,223 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:29,223 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/cef962fd4465424097d4d7182de070ea is 50, key is test_row_0/C:col10/1731477148718/Put/seqid=0 2024-11-13T05:52:29,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742328_1504 (size=12629) 2024-11-13T05:52:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:29,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:52:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:29,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/60613bd72ed84e2b80eab4659c2ebd2d is 50, key is test_row_0/A:col10/1731477148753/Put/seqid=0 2024-11-13T05:52:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742329_1505 (size=14541) 2024-11-13T05:52:29,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477149501, max timestamp=1731477149539Num uniques 859; 2024-11-13T05:52:29,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:29,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477149501, max timestamp=1731477149539Num uniques 859; 2024-11-13T05:52:29,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:29,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477149501, max timestamp=1731477149539Num uniques 859; 2024-11-13T05:52:29,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:29,630 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/cef962fd4465424097d4d7182de070ea as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/cef962fd4465424097d4d7182de070ea 2024-11-13T05:52:29,633 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into cef962fd4465424097d4d7182de070ea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:29,633 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:29,633 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=12, startTime=1731477149176; duration=0sec 2024-11-13T05:52:29,633 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:29,633 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:29,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477209695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:29,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477209812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:29,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:29,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/60613bd72ed84e2b80eab4659c2ebd2d 2024-11-13T05:52:29,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e094378aa4b541449533a9a8f7f2e2c3 is 50, key is test_row_0/B:col10/1731477148753/Put/seqid=0 2024-11-13T05:52:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742330_1506 (size=12151) 2024-11-13T05:52:29,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e094378aa4b541449533a9a8f7f2e2c3 2024-11-13T05:52:29,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/632e86560a574776b837d5f6450be940 is 50, key is test_row_0/C:col10/1731477148753/Put/seqid=0 2024-11-13T05:52:29,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742331_1507 (size=12151) 2024-11-13T05:52:30,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477210031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/632e86560a574776b837d5f6450be940 2024-11-13T05:52:30,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/60613bd72ed84e2b80eab4659c2ebd2d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/60613bd72ed84e2b80eab4659c2ebd2d 2024-11-13T05:52:30,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/60613bd72ed84e2b80eab4659c2ebd2d, entries=200, sequenceid=225, filesize=14.2 K 2024-11-13T05:52:30,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/e094378aa4b541449533a9a8f7f2e2c3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e094378aa4b541449533a9a8f7f2e2c3 2024-11-13T05:52:30,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e094378aa4b541449533a9a8f7f2e2c3, entries=150, sequenceid=225, filesize=11.9 K 2024-11-13T05:52:30,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/632e86560a574776b837d5f6450be940 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/632e86560a574776b837d5f6450be940 2024-11-13T05:52:30,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/632e86560a574776b837d5f6450be940, entries=150, sequenceid=225, filesize=11.9 K 2024-11-13T05:52:30,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d5b6967321153590da5042dc4609c1e3 in 866ms, sequenceid=225, compaction requested=false 2024-11-13T05:52:30,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:30,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-13T05:52:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:30,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:30,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/a79d0586a7dd4b618da2711459580b50 is 50, key is test_row_0/A:col10/1731477150358/Put/seqid=0 2024-11-13T05:52:30,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742332_1508 (size=14691) 2024-11-13T05:52:30,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477210410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T05:52:30,491 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:30,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T05:52:30,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T05:52:30,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:30,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477210521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T05:52:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51682 deadline: 1731477210611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,615 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T05:52:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:30,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51588 deadline: 1731477210641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51624 deadline: 1731477210642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1731477210642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,653 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,653 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,653 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in d5b6967321153590da5042dc4609c1e3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477210732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:30,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/a79d0586a7dd4b618da2711459580b50 2024-11-13T05:52:30,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7023c2736dca4400aa55d06a8c7a8257 is 50, key is test_row_0/B:col10/1731477150358/Put/seqid=0 2024-11-13T05:52:30,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742333_1509 (size=12251) 2024-11-13T05:52:30,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T05:52:30,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:30,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:30,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:30,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T05:52:30,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T05:52:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:30,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477211051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:31,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T05:52:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T05:52:31,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7023c2736dca4400aa55d06a8c7a8257 2024-11-13T05:52:31,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/fca91299cf784dd98e917523abe76f3c is 50, key is test_row_0/C:col10/1731477150358/Put/seqid=0 2024-11-13T05:52:31,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742334_1510 (size=12251) 2024-11-13T05:52:31,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/fca91299cf784dd98e917523abe76f3c 2024-11-13T05:52:31,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/a79d0586a7dd4b618da2711459580b50 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/a79d0586a7dd4b618da2711459580b50 2024-11-13T05:52:31,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/a79d0586a7dd4b618da2711459580b50, entries=200, sequenceid=264, filesize=14.3 K 2024-11-13T05:52:31,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7023c2736dca4400aa55d06a8c7a8257 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7023c2736dca4400aa55d06a8c7a8257 2024-11-13T05:52:31,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7023c2736dca4400aa55d06a8c7a8257, entries=150, sequenceid=264, filesize=12.0 K 2024-11-13T05:52:31,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/fca91299cf784dd98e917523abe76f3c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/fca91299cf784dd98e917523abe76f3c 2024-11-13T05:52:31,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/fca91299cf784dd98e917523abe76f3c, entries=150, sequenceid=264, filesize=12.0 K 2024-11-13T05:52:31,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for d5b6967321153590da5042dc4609c1e3 in 857ms, sequenceid=264, compaction requested=true 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:31,219 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:31,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:31,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:31,220 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:31,220 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:31,220 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,220 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/3cffbc7390534d688e697a683f79e1d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/60613bd72ed84e2b80eab4659c2ebd2d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/a79d0586a7dd4b618da2711459580b50] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=40.9 K 2024-11-13T05:52:31,221 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cffbc7390534d688e697a683f79e1d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731477148344 2024-11-13T05:52:31,221 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60613bd72ed84e2b80eab4659c2ebd2d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731477148753 2024-11-13T05:52:31,221 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79d0586a7dd4b618da2711459580b50, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731477149592 2024-11-13T05:52:31,221 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:31,222 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:31,222 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,222 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/546e7ad149454a3593ca5a7d7c82da16, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e094378aa4b541449533a9a8f7f2e2c3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7023c2736dca4400aa55d06a8c7a8257] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.2 K 2024-11-13T05:52:31,224 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 546e7ad149454a3593ca5a7d7c82da16, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731477148344 2024-11-13T05:52:31,225 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e094378aa4b541449533a9a8f7f2e2c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731477148753 2024-11-13T05:52:31,225 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7023c2736dca4400aa55d06a8c7a8257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731477149592 2024-11-13T05:52:31,245 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:31,246 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/09701e6baf3245309c68a9dc2b74c7d4 is 50, key is test_row_0/A:col10/1731477150358/Put/seqid=0 2024-11-13T05:52:31,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,256 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:31,260 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:31,261 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/bf24832e5160438084c23e08877b7077 is 50, key is test_row_0/B:col10/1731477150358/Put/seqid=0 2024-11-13T05:52:31,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742335_1511 (size=12831) 2024-11-13T05:52:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d30915bf776f46f7a11bd76d625d7dfa is 50, key is test_row_0/A:col10/1731477150391/Put/seqid=0 2024-11-13T05:52:31,274 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/09701e6baf3245309c68a9dc2b74c7d4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/09701e6baf3245309c68a9dc2b74c7d4 2024-11-13T05:52:31,285 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into 09701e6baf3245309c68a9dc2b74c7d4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:31,285 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:31,285 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=13, startTime=1731477151219; duration=0sec 2024-11-13T05:52:31,285 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:31,285 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:31,285 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:31,288 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:31,288 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:31,288 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,288 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/cef962fd4465424097d4d7182de070ea, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/632e86560a574776b837d5f6450be940, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/fca91299cf784dd98e917523abe76f3c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.2 K 2024-11-13T05:52:31,288 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef962fd4465424097d4d7182de070ea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731477148344 2024-11-13T05:52:31,288 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 632e86560a574776b837d5f6450be940, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731477148753 2024-11-13T05:52:31,289 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca91299cf784dd98e917523abe76f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731477149592 2024-11-13T05:52:31,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742336_1512 (size=12831) 2024-11-13T05:52:31,299 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/bf24832e5160438084c23e08877b7077 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/bf24832e5160438084c23e08877b7077 2024-11-13T05:52:31,301 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:31,302 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/8d72e7440dcf40eaad494c45a569fd2c is 50, key is test_row_0/C:col10/1731477150358/Put/seqid=0 2024-11-13T05:52:31,315 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into bf24832e5160438084c23e08877b7077(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:31,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742337_1513 (size=12301) 2024-11-13T05:52:31,315 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:31,315 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=13, startTime=1731477151219; duration=0sec 2024-11-13T05:52:31,316 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:31,316 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:31,316 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d30915bf776f46f7a11bd76d625d7dfa 2024-11-13T05:52:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742338_1514 (size=12831) 2024-11-13T05:52:31,331 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/8d72e7440dcf40eaad494c45a569fd2c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/8d72e7440dcf40eaad494c45a569fd2c 2024-11-13T05:52:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7c62438cced3460cac843f15c4bf7675 is 50, key is test_row_0/B:col10/1731477150391/Put/seqid=0 2024-11-13T05:52:31,339 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 8d72e7440dcf40eaad494c45a569fd2c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:31,339 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:31,339 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=13, startTime=1731477151219; duration=0sec 2024-11-13T05:52:31,339 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:31,339 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:31,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742339_1515 (size=12301) 2024-11-13T05:52:31,345 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7c62438cced3460cac843f15c4bf7675 2024-11-13T05:52:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/118937e1faf9431d90066c964b0a7eae is 50, key is test_row_0/C:col10/1731477150391/Put/seqid=0 2024-11-13T05:52:31,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742340_1516 (size=12301) 2024-11-13T05:52:31,400 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/118937e1faf9431d90066c964b0a7eae 2024-11-13T05:52:31,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d30915bf776f46f7a11bd76d625d7dfa as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d30915bf776f46f7a11bd76d625d7dfa 2024-11-13T05:52:31,407 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d30915bf776f46f7a11bd76d625d7dfa, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T05:52:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7c62438cced3460cac843f15c4bf7675 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7c62438cced3460cac843f15c4bf7675 2024-11-13T05:52:31,412 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7c62438cced3460cac843f15c4bf7675, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T05:52:31,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/118937e1faf9431d90066c964b0a7eae as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/118937e1faf9431d90066c964b0a7eae 2024-11-13T05:52:31,419 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/118937e1faf9431d90066c964b0a7eae, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T05:52:31,420 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for d5b6967321153590da5042dc4609c1e3 in 163ms, sequenceid=273, compaction requested=false 2024-11-13T05:52:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T05:52:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T05:52:31,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T05:52:31,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 928 msec 2024-11-13T05:52:31,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 932 msec 2024-11-13T05:52:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:31,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:52:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:31,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/073084d62dd348a2b089622ebf479516 is 50, key is test_row_0/A:col10/1731477151577/Put/seqid=0 2024-11-13T05:52:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742341_1517 (size=14741) 2024-11-13T05:52:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T05:52:31,632 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:31,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T05:52:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:31,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:31,634 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:31,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477151652, max timestamp=1731477151722Num uniques 859; 2024-11-13T05:52:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477151652, max timestamp=1731477151722Num uniques 859; 2024-11-13T05:52:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477151652, max timestamp=1731477151722Num uniques 859; 2024-11-13T05:52:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:31,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477211857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:31,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:31,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:31,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:31,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:31,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:31,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477211971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:32,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/073084d62dd348a2b089622ebf479516 2024-11-13T05:52:32,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/1b4f13826ac54e1a8928db750b56446b is 50, key is test_row_0/B:col10/1731477151577/Put/seqid=0 2024-11-13T05:52:32,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742342_1518 (size=12301) 2024-11-13T05:52:32,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/1b4f13826ac54e1a8928db750b56446b 2024-11-13T05:52:32,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d32f2013aba845ab98e60ef98fcacda5 is 50, key is test_row_0/C:col10/1731477151577/Put/seqid=0 2024-11-13T05:52:32,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742343_1519 (size=12301) 2024-11-13T05:52:32,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:32,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477212192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:32,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:32,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d32f2013aba845ab98e60ef98fcacda5 2024-11-13T05:52:32,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/073084d62dd348a2b089622ebf479516 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/073084d62dd348a2b089622ebf479516 2024-11-13T05:52:32,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/073084d62dd348a2b089622ebf479516, entries=200, sequenceid=286, filesize=14.4 K 2024-11-13T05:52:32,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/1b4f13826ac54e1a8928db750b56446b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1b4f13826ac54e1a8928db750b56446b 2024-11-13T05:52:32,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1b4f13826ac54e1a8928db750b56446b, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T05:52:32,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d32f2013aba845ab98e60ef98fcacda5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d32f2013aba845ab98e60ef98fcacda5 2024-11-13T05:52:32,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d32f2013aba845ab98e60ef98fcacda5, entries=150, sequenceid=286, filesize=12.0 K 2024-11-13T05:52:32,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d5b6967321153590da5042dc4609c1e3 in 901ms, sequenceid=286, compaction requested=true 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:32,494 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:32,494 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:32,495 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,495 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,495 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/bf24832e5160438084c23e08877b7077, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7c62438cced3460cac843f15c4bf7675, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1b4f13826ac54e1a8928db750b56446b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.6 K 2024-11-13T05:52:32,495 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/09701e6baf3245309c68a9dc2b74c7d4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d30915bf776f46f7a11bd76d625d7dfa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/073084d62dd348a2b089622ebf479516] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=38.9 K 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting bf24832e5160438084c23e08877b7077, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731477149592 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09701e6baf3245309c68a9dc2b74c7d4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731477149592 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c62438cced3460cac843f15c4bf7675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731477150391 2024-11-13T05:52:32,495 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30915bf776f46f7a11bd76d625d7dfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731477150391 2024-11-13T05:52:32,496 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4f13826ac54e1a8928db750b56446b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731477151577 2024-11-13T05:52:32,496 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073084d62dd348a2b089622ebf479516, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731477151577 2024-11-13T05:52:32,500 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:32,500 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:32,500 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/e51af00b4521457f9a3c6b212b58ab68 is 50, key is test_row_0/A:col10/1731477151577/Put/seqid=0 2024-11-13T05:52:32,500 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9f7093424adb493191751f06a2bee1ff is 50, key is test_row_0/B:col10/1731477151577/Put/seqid=0 2024-11-13T05:52:32,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742344_1520 (size=12983) 2024-11-13T05:52:32,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742345_1521 (size=12983) 2024-11-13T05:52:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:32,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-13T05:52:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:32,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/2b56b815c9cd41998291c3fee2c1a9d9 is 50, key is test_row_0/A:col10/1731477151856/Put/seqid=0 2024-11-13T05:52:32,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742346_1522 (size=14741) 2024-11-13T05:52:32,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477212571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:32,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477212691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:32,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:32,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:32,908 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9f7093424adb493191751f06a2bee1ff as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9f7093424adb493191751f06a2bee1ff 2024-11-13T05:52:32,910 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/e51af00b4521457f9a3c6b212b58ab68 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e51af00b4521457f9a3c6b212b58ab68 2024-11-13T05:52:32,912 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into 9f7093424adb493191751f06a2bee1ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:32,912 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:32,912 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=13, startTime=1731477152494; duration=0sec 2024-11-13T05:52:32,912 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:32,912 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:32,912 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:32,912 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:32,913 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:32,913 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:32,913 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/8d72e7440dcf40eaad494c45a569fd2c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/118937e1faf9431d90066c964b0a7eae, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d32f2013aba845ab98e60ef98fcacda5] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.6 K 2024-11-13T05:52:32,913 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d72e7440dcf40eaad494c45a569fd2c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731477149592 2024-11-13T05:52:32,913 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 118937e1faf9431d90066c964b0a7eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731477150391 2024-11-13T05:52:32,913 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d32f2013aba845ab98e60ef98fcacda5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731477151577 2024-11-13T05:52:32,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477212911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:32,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:32,914 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into e51af00b4521457f9a3c6b212b58ab68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:32,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:32,914 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=13, startTime=1731477152494; duration=0sec 2024-11-13T05:52:32,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:32,914 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:32,918 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:32,919 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/71bbdd6839a64c4185eb330b9a936f90 is 50, key is test_row_0/C:col10/1731477151577/Put/seqid=0 2024-11-13T05:52:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742347_1523 (size=12983) 2024-11-13T05:52:32,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/2b56b815c9cd41998291c3fee2c1a9d9 2024-11-13T05:52:32,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a01b88ba720b45af87129c1b2f8c16ff is 50, key is test_row_0/B:col10/1731477151856/Put/seqid=0 2024-11-13T05:52:32,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742348_1524 (size=12301) 2024-11-13T05:52:32,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a01b88ba720b45af87129c1b2f8c16ff 2024-11-13T05:52:32,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5fea9dd4050a4ba9bbbdab4775784929 is 50, key is test_row_0/C:col10/1731477151856/Put/seqid=0 2024-11-13T05:52:32,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742349_1525 (size=12301) 2024-11-13T05:52:33,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477213222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:33,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:33,326 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/71bbdd6839a64c4185eb330b9a936f90 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/71bbdd6839a64c4185eb330b9a936f90 2024-11-13T05:52:33,330 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 71bbdd6839a64c4185eb330b9a936f90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:33,330 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:33,330 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=13, startTime=1731477152494; duration=0sec 2024-11-13T05:52:33,330 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:33,330 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:33,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5fea9dd4050a4ba9bbbdab4775784929 2024-11-13T05:52:33,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/2b56b815c9cd41998291c3fee2c1a9d9 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/2b56b815c9cd41998291c3fee2c1a9d9 2024-11-13T05:52:33,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/2b56b815c9cd41998291c3fee2c1a9d9, entries=200, sequenceid=323, filesize=14.4 K 2024-11-13T05:52:33,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a01b88ba720b45af87129c1b2f8c16ff as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a01b88ba720b45af87129c1b2f8c16ff 2024-11-13T05:52:33,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a01b88ba720b45af87129c1b2f8c16ff, entries=150, sequenceid=323, filesize=12.0 K 2024-11-13T05:52:33,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5fea9dd4050a4ba9bbbdab4775784929 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5fea9dd4050a4ba9bbbdab4775784929 2024-11-13T05:52:33,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5fea9dd4050a4ba9bbbdab4775784929, entries=150, sequenceid=323, filesize=12.0 K 2024-11-13T05:52:33,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for d5b6967321153590da5042dc4609c1e3 in 845ms, sequenceid=323, compaction requested=false 2024-11-13T05:52:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:33,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T05:52:33,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:33,465 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T05:52:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5a0c48514fcc41eda72d3ff8738201c5 is 50, key is test_row_0/A:col10/1731477152550/Put/seqid=0 2024-11-13T05:52:33,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742350_1526 (size=9857) 2024-11-13T05:52:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:33,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477153806, max timestamp=1731477153825Num uniques 859; 2024-11-13T05:52:33,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477153806, max timestamp=1731477153825Num uniques 859; 2024-11-13T05:52:33,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477153806, max timestamp=1731477153825Num uniques 859; 2024-11-13T05:52:33,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:33,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:33,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:33,874 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5a0c48514fcc41eda72d3ff8738201c5 2024-11-13T05:52:33,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/12a7f509b42f4c9b99f3ec6741319b5d is 50, key is test_row_0/B:col10/1731477152550/Put/seqid=0 2024-11-13T05:52:33,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742351_1527 (size=9857) 2024-11-13T05:52:33,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477153888, max timestamp=1731477153940Num uniques 1018; 2024-11-13T05:52:33,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T05:52:33,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477153888, max timestamp=1731477153940Num uniques 1018; 2024-11-13T05:52:33,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T05:52:33,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731477153888, max timestamp=1731477153940Num uniques 1018; 2024-11-13T05:52:33,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-13T05:52:34,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731477154015, max timestamp=1731477154062Num uniques 1027; 2024-11-13T05:52:34,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T05:52:34,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731477154015, max timestamp=1731477154062Num uniques 1027; 2024-11-13T05:52:34,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T05:52:34,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731477154015, max timestamp=1731477154062Num uniques 1027; 2024-11-13T05:52:34,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-13T05:52:34,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477214183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:34,283 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/12a7f509b42f4c9b99f3ec6741319b5d 2024-11-13T05:52:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/a20374d1fc934b4ba702a1854311c19f is 50, key is test_row_0/C:col10/1731477152550/Put/seqid=0 2024-11-13T05:52:34,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742352_1528 (size=9857) 2024-11-13T05:52:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477214301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:34,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477214522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:34,692 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/a20374d1fc934b4ba702a1854311c19f 2024-11-13T05:52:34,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/5a0c48514fcc41eda72d3ff8738201c5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a0c48514fcc41eda72d3ff8738201c5 2024-11-13T05:52:34,698 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a0c48514fcc41eda72d3ff8738201c5, entries=100, sequenceid=335, filesize=9.6 K 2024-11-13T05:52:34,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/12a7f509b42f4c9b99f3ec6741319b5d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/12a7f509b42f4c9b99f3ec6741319b5d 2024-11-13T05:52:34,701 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/12a7f509b42f4c9b99f3ec6741319b5d, entries=100, sequenceid=335, filesize=9.6 K 2024-11-13T05:52:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/a20374d1fc934b4ba702a1854311c19f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a20374d1fc934b4ba702a1854311c19f 2024-11-13T05:52:34,704 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a20374d1fc934b4ba702a1854311c19f, entries=100, sequenceid=335, filesize=9.6 K 2024-11-13T05:52:34,704 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for d5b6967321153590da5042dc4609c1e3 in 1239ms, sequenceid=335, compaction requested=true 2024-11-13T05:52:34,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:34,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:34,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T05:52:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T05:52:34,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T05:52:34,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0710 sec 2024-11-13T05:52:34,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 3.0760 sec 2024-11-13T05:52:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:34,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=147.60 KB heapSize=419.16 KB 2024-11-13T05:52:34,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:34,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:34,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:34,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:34,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:34,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-13T05:52:34,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/628de2cda2924bd094fd91e6b0b3b8e5 is 50, key is test_row_0/A:col10/1731477154181/Put/seqid=0 2024-11-13T05:52:34,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742353_1529 (size=14741) 2024-11-13T05:52:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477214878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:34,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477214991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477215201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:35,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/628de2cda2924bd094fd91e6b0b3b8e5 2024-11-13T05:52:35,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/8678934c9c424ac199042dcc3df87c53 is 50, key is test_row_0/B:col10/1731477154181/Put/seqid=0 2024-11-13T05:52:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742354_1530 (size=12301) 2024-11-13T05:52:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477215511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:35,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/8678934c9c424ac199042dcc3df87c53 2024-11-13T05:52:35,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d67706eeb1e744118d8e1ab01b269a3a is 50, key is test_row_0/C:col10/1731477154181/Put/seqid=0 2024-11-13T05:52:35,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742355_1531 (size=12301) 2024-11-13T05:52:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T05:52:35,802 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:35,802 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-13T05:52:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T05:52:35,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:35,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:35,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T05:52:35,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T05:52:35,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:35,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:35,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:35,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:35,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:36,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477216021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:36,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d67706eeb1e744118d8e1ab01b269a3a 2024-11-13T05:52:36,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/628de2cda2924bd094fd91e6b0b3b8e5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/628de2cda2924bd094fd91e6b0b3b8e5 2024-11-13T05:52:36,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/628de2cda2924bd094fd91e6b0b3b8e5, entries=200, sequenceid=407, filesize=14.4 K 2024-11-13T05:52:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/8678934c9c424ac199042dcc3df87c53 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/8678934c9c424ac199042dcc3df87c53 2024-11-13T05:52:36,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/8678934c9c424ac199042dcc3df87c53, entries=150, sequenceid=407, filesize=12.0 K 2024-11-13T05:52:36,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d67706eeb1e744118d8e1ab01b269a3a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d67706eeb1e744118d8e1ab01b269a3a 2024-11-13T05:52:36,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d67706eeb1e744118d8e1ab01b269a3a, entries=150, sequenceid=407, filesize=12.0 K 2024-11-13T05:52:36,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.11 KB/429168, currentSize=33.54 KB/34350 for d5b6967321153590da5042dc4609c1e3 in 1244ms, sequenceid=407, compaction requested=true 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:36,082 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:36,082 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:36,083 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:36,083 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:36,083 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:36,083 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:36,083 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:36,083 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:36,083 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9f7093424adb493191751f06a2bee1ff, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a01b88ba720b45af87129c1b2f8c16ff, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/12a7f509b42f4c9b99f3ec6741319b5d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/8678934c9c424ac199042dcc3df87c53] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=46.3 K 2024-11-13T05:52:36,083 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e51af00b4521457f9a3c6b212b58ab68, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/2b56b815c9cd41998291c3fee2c1a9d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a0c48514fcc41eda72d3ff8738201c5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/628de2cda2924bd094fd91e6b0b3b8e5] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=51.1 K 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7093424adb493191751f06a2bee1ff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731477151577 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting e51af00b4521457f9a3c6b212b58ab68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731477151577 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a01b88ba720b45af87129c1b2f8c16ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731477151779 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b56b815c9cd41998291c3fee2c1a9d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731477151779 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 12a7f509b42f4c9b99f3ec6741319b5d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731477152550 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8678934c9c424ac199042dcc3df87c53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731477154108 2024-11-13T05:52:36,084 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0c48514fcc41eda72d3ff8738201c5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731477152550 2024-11-13T05:52:36,085 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628de2cda2924bd094fd91e6b0b3b8e5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731477154108 2024-11-13T05:52:36,092 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:36,092 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7f2bbd5d2fa7409db5df08288df2fe3c is 50, key is test_row_0/B:col10/1731477154181/Put/seqid=0 2024-11-13T05:52:36,103 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:36,104 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/bdb2bddfa549469e8086d4d56b44b111 is 50, key is test_row_0/A:col10/1731477154181/Put/seqid=0 2024-11-13T05:52:36,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742356_1532 (size=13119) 2024-11-13T05:52:36,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742357_1533 (size=13119) 2024-11-13T05:52:36,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T05:52:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:36,109 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T05:52:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:36,113 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/bdb2bddfa549469e8086d4d56b44b111 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/bdb2bddfa549469e8086d4d56b44b111 2024-11-13T05:52:36,117 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into bdb2bddfa549469e8086d4d56b44b111(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:36,117 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:36,117 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=12, startTime=1731477156082; duration=0sec 2024-11-13T05:52:36,117 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:36,117 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:36,117 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:52:36,118 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T05:52:36,118 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:36,118 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:36,118 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/71bbdd6839a64c4185eb330b9a936f90, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5fea9dd4050a4ba9bbbdab4775784929, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a20374d1fc934b4ba702a1854311c19f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d67706eeb1e744118d8e1ab01b269a3a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=46.3 K 2024-11-13T05:52:36,119 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71bbdd6839a64c4185eb330b9a936f90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731477151577 2024-11-13T05:52:36,119 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fea9dd4050a4ba9bbbdab4775784929, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731477151779 2024-11-13T05:52:36,119 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a20374d1fc934b4ba702a1854311c19f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731477152550 2024-11-13T05:52:36,119 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d67706eeb1e744118d8e1ab01b269a3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731477154108 2024-11-13T05:52:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T05:52:36,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/007e9c55f1ac40d9b8597d3ffd8a8f6b is 50, key is test_row_0/A:col10/1731477154876/Put/seqid=0 2024-11-13T05:52:36,130 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:36,130 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/59e8def5241046329b89d98b61eb7b00 is 50, key is test_row_0/C:col10/1731477154181/Put/seqid=0 2024-11-13T05:52:36,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742358_1534 (size=9857) 2024-11-13T05:52:36,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742359_1535 (size=13119) 2024-11-13T05:52:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T05:52:36,509 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/7f2bbd5d2fa7409db5df08288df2fe3c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7f2bbd5d2fa7409db5df08288df2fe3c 2024-11-13T05:52:36,513 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into 7f2bbd5d2fa7409db5df08288df2fe3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:36,513 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:36,513 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=12, startTime=1731477156082; duration=0sec 2024-11-13T05:52:36,513 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:36,513 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:36,540 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/007e9c55f1ac40d9b8597d3ffd8a8f6b 2024-11-13T05:52:36,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0cb32939432344e98b658a18adabc62c is 50, key is test_row_0/B:col10/1731477154876/Put/seqid=0 2024-11-13T05:52:36,547 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/59e8def5241046329b89d98b61eb7b00 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/59e8def5241046329b89d98b61eb7b00 2024-11-13T05:52:36,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742360_1536 (size=9857) 2024-11-13T05:52:36,552 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 59e8def5241046329b89d98b61eb7b00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:36,552 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:36,552 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=12, startTime=1731477156082; duration=0sec 2024-11-13T05:52:36,552 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:36,552 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:36,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T05:52:36,950 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0cb32939432344e98b658a18adabc62c 2024-11-13T05:52:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/a0225c8098334742a9ef1653b8f20ae7 is 50, key is test_row_0/C:col10/1731477154876/Put/seqid=0 2024-11-13T05:52:36,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742361_1537 (size=9857) 2024-11-13T05:52:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:37,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477157078, max timestamp=1731477157123Num uniques 859; 2024-11-13T05:52:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477157078, max timestamp=1731477157123Num uniques 859; 2024-11-13T05:52:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477157078, max timestamp=1731477157123Num uniques 859; 2024-11-13T05:52:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:37,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731477157196, max timestamp=1731477157249Num uniques 1068; 2024-11-13T05:52:37,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731477157196, max timestamp=1731477157249Num uniques 1068; 2024-11-13T05:52:37,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731477157196, max timestamp=1731477157249Num uniques 1068; 2024-11-13T05:52:37,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T05:52:37,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T05:52:37,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-13T05:52:37,359 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/a0225c8098334742a9ef1653b8f20ae7 2024-11-13T05:52:37,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/007e9c55f1ac40d9b8597d3ffd8a8f6b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/007e9c55f1ac40d9b8597d3ffd8a8f6b 2024-11-13T05:52:37,364 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/007e9c55f1ac40d9b8597d3ffd8a8f6b, entries=100, sequenceid=415, filesize=9.6 K 2024-11-13T05:52:37,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0cb32939432344e98b658a18adabc62c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0cb32939432344e98b658a18adabc62c 2024-11-13T05:52:37,368 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0cb32939432344e98b658a18adabc62c, entries=100, sequenceid=415, filesize=9.6 K 2024-11-13T05:52:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/a0225c8098334742a9ef1653b8f20ae7 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a0225c8098334742a9ef1653b8f20ae7 2024-11-13T05:52:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a0225c8098334742a9ef1653b8f20ae7, entries=100, sequenceid=415, filesize=9.6 K 2024-11-13T05:52:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T05:52:37,372 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=134.18 KB/137400 for d5b6967321153590da5042dc4609c1e3 in 1262ms, sequenceid=415, compaction requested=false 2024-11-13T05:52:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:37,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-13T05:52:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-13T05:52:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:37,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-13T05:52:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:37,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:37,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T05:52:37,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-13T05:52:37,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5740 sec 2024-11-13T05:52:37,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d7b64965744044cda1de1084dbc1c44c is 50, key is test_row_0/A:col10/1731477157372/Put/seqid=0 2024-11-13T05:52:37,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742362_1538 (size=14741) 2024-11-13T05:52:37,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d7b64965744044cda1de1084dbc1c44c 2024-11-13T05:52:37,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/14390f8dc28341cbaeafa635ed732dd0 is 50, key is test_row_0/B:col10/1731477157372/Put/seqid=0 2024-11-13T05:52:37,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742363_1539 (size=12301) 2024-11-13T05:52:37,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 369 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477217415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:37,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 370 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477217531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:37,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 371 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477217751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:37,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/14390f8dc28341cbaeafa635ed732dd0 2024-11-13T05:52:37,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5588e7cd92f8461e944a544271357848 is 50, key is test_row_0/C:col10/1731477157372/Put/seqid=0 2024-11-13T05:52:37,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742364_1540 (size=12301) 2024-11-13T05:52:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T05:52:37,952 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:37,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-13T05:52:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T05:52:37,962 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:37,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:37,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T05:52:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 372 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477218062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:38,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T05:52:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:38,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5588e7cd92f8461e944a544271357848 2024-11-13T05:52:38,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/d7b64965744044cda1de1084dbc1c44c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d7b64965744044cda1de1084dbc1c44c 2024-11-13T05:52:38,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d7b64965744044cda1de1084dbc1c44c, entries=200, sequenceid=472, filesize=14.4 K 2024-11-13T05:52:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/14390f8dc28341cbaeafa635ed732dd0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/14390f8dc28341cbaeafa635ed732dd0 2024-11-13T05:52:38,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/14390f8dc28341cbaeafa635ed732dd0, entries=150, sequenceid=472, filesize=12.0 K 2024-11-13T05:52:38,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/5588e7cd92f8461e944a544271357848 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5588e7cd92f8461e944a544271357848 2024-11-13T05:52:38,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5588e7cd92f8461e944a544271357848, entries=150, sequenceid=472, filesize=12.0 K 2024-11-13T05:52:38,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for d5b6967321153590da5042dc4609c1e3 in 861ms, sequenceid=472, compaction requested=true 2024-11-13T05:52:38,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:38,234 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:38,241 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:38,241 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:38,241 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,241 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/bdb2bddfa549469e8086d4d56b44b111, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/007e9c55f1ac40d9b8597d3ffd8a8f6b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d7b64965744044cda1de1084dbc1c44c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.8 K 2024-11-13T05:52:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:38,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:38,241 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb2bddfa549469e8086d4d56b44b111, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731477154108 2024-11-13T05:52:38,242 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 007e9c55f1ac40d9b8597d3ffd8a8f6b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731477154865 2024-11-13T05:52:38,242 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7b64965744044cda1de1084dbc1c44c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731477157332 2024-11-13T05:52:38,243 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:38,246 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:38,246 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:38,246 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,247 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7f2bbd5d2fa7409db5df08288df2fe3c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0cb32939432344e98b658a18adabc62c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/14390f8dc28341cbaeafa635ed732dd0] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=34.5 K 2024-11-13T05:52:38,247 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2bbd5d2fa7409db5df08288df2fe3c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731477154108 2024-11-13T05:52:38,248 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb32939432344e98b658a18adabc62c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731477154865 2024-11-13T05:52:38,248 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 14390f8dc28341cbaeafa635ed732dd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731477157332 2024-11-13T05:52:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:38,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:38,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:38,266 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#459 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:38,267 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c4fca625161b4a74ae85010af2eb73be is 50, key is test_row_0/A:col10/1731477157372/Put/seqid=0 2024-11-13T05:52:38,271 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:38,272 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a9bed02324a1499e9320a983370d5c96 is 50, key is test_row_0/B:col10/1731477157372/Put/seqid=0 2024-11-13T05:52:38,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T05:52:38,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,280 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-13T05:52:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:38,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T05:52:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c54f5d0acbc44cb4bc79e7ac78ce878e is 50, key is test_row_0/A:col10/1731477157399/Put/seqid=0 2024-11-13T05:52:38,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742365_1541 (size=13221) 2024-11-13T05:52:38,346 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c4fca625161b4a74ae85010af2eb73be as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c4fca625161b4a74ae85010af2eb73be 2024-11-13T05:52:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742366_1542 (size=13221) 2024-11-13T05:52:38,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742367_1543 (size=12301) 2024-11-13T05:52:38,355 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c54f5d0acbc44cb4bc79e7ac78ce878e 2024-11-13T05:52:38,357 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into c4fca625161b4a74ae85010af2eb73be(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:38,357 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:38,357 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=13, startTime=1731477158234; duration=0sec 2024-11-13T05:52:38,358 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:38,358 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:38,358 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:38,365 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:38,365 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:38,365 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,365 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/59e8def5241046329b89d98b61eb7b00, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a0225c8098334742a9ef1653b8f20ae7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5588e7cd92f8461e944a544271357848] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=34.5 K 2024-11-13T05:52:38,367 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e8def5241046329b89d98b61eb7b00, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731477154108 2024-11-13T05:52:38,369 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0225c8098334742a9ef1653b8f20ae7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731477154865 2024-11-13T05:52:38,369 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a9bed02324a1499e9320a983370d5c96 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a9bed02324a1499e9320a983370d5c96 2024-11-13T05:52:38,369 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5588e7cd92f8461e944a544271357848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731477157332 2024-11-13T05:52:38,378 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into a9bed02324a1499e9320a983370d5c96(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:38,378 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:38,378 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=13, startTime=1731477158242; duration=0sec 2024-11-13T05:52:38,378 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:38,378 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9faaa59754e24d388893ae40c6134c81 is 50, key is test_row_0/B:col10/1731477157399/Put/seqid=0 2024-11-13T05:52:38,399 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:38,399 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/ae45c699d344492b837e4332198f4100 is 50, key is test_row_0/C:col10/1731477157372/Put/seqid=0 2024-11-13T05:52:38,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742368_1544 (size=12301) 2024-11-13T05:52:38,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742369_1545 (size=13221) 2024-11-13T05:52:38,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T05:52:38,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477158672, max timestamp=1731477158755Num uniques 859; 2024-11-13T05:52:38,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477158672, max timestamp=1731477158755Num uniques 859; 2024-11-13T05:52:38,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:38,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:38,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477158672, max timestamp=1731477158755Num uniques 859; 2024-11-13T05:52:38,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:38,836 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9faaa59754e24d388893ae40c6134c81 2024-11-13T05:52:38,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/132db77eb1b1427cad4258fbdc55ee98 is 50, key is test_row_0/C:col10/1731477157399/Put/seqid=0 2024-11-13T05:52:38,864 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/ae45c699d344492b837e4332198f4100 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ae45c699d344492b837e4332198f4100 2024-11-13T05:52:38,870 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into ae45c699d344492b837e4332198f4100(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:38,870 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:38,870 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=13, startTime=1731477158257; duration=0sec 2024-11-13T05:52:38,870 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:38,870 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:38,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742370_1546 (size=12301) 2024-11-13T05:52:38,896 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/132db77eb1b1427cad4258fbdc55ee98 2024-11-13T05:52:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c54f5d0acbc44cb4bc79e7ac78ce878e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c54f5d0acbc44cb4bc79e7ac78ce878e 2024-11-13T05:52:38,905 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c54f5d0acbc44cb4bc79e7ac78ce878e, entries=150, sequenceid=481, filesize=12.0 K 2024-11-13T05:52:38,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/9faaa59754e24d388893ae40c6134c81 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9faaa59754e24d388893ae40c6134c81 2024-11-13T05:52:38,909 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9faaa59754e24d388893ae40c6134c81, entries=150, sequenceid=481, filesize=12.0 K 2024-11-13T05:52:38,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/132db77eb1b1427cad4258fbdc55ee98 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/132db77eb1b1427cad4258fbdc55ee98 2024-11-13T05:52:38,914 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/132db77eb1b1427cad4258fbdc55ee98, entries=150, sequenceid=481, filesize=12.0 K 2024-11-13T05:52:38,915 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=107.34 KB/109920 for d5b6967321153590da5042dc4609c1e3 in 635ms, sequenceid=481, compaction requested=false 2024-11-13T05:52:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-13T05:52:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-13T05:52:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:38,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-11-13T05:52:38,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:38,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:38,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:38,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:38,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:38,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:38,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T05:52:38,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 954 msec 2024-11-13T05:52:38,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 964 msec 2024-11-13T05:52:38,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/4448520cf95044b6bbabe0796b7df27d is 50, key is test_row_0/A:col10/1731477158888/Put/seqid=0 2024-11-13T05:52:38,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742371_1547 (size=14741) 2024-11-13T05:52:39,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 414 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477219081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T05:52:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:39,102 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:39,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-13T05:52:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T05:52:39,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:39,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:39,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T05:52:39,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 415 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477219214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:39,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T05:52:39,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:39,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/4448520cf95044b6bbabe0796b7df27d 2024-11-13T05:52:39,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/79f798c526674bddb9a9994f0e8b780c is 50, key is test_row_0/B:col10/1731477158888/Put/seqid=0 2024-11-13T05:52:39,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742372_1548 (size=12301) 2024-11-13T05:52:39,408 INFO [Thread-2064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2064. 2024-11-13T05:52:39,408 DEBUG [Thread-2064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:39,408 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,408 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:39,410 INFO [Thread-2068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2068. 2024-11-13T05:52:39,411 DEBUG [Thread-2068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:39,411 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,411 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:39,415 INFO [Thread-2062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2062. 2024-11-13T05:52:39,415 DEBUG [Thread-2062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:39,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T05:52:39,415 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,415 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T05:52:39,426 INFO [Thread-2066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2066. 2024-11-13T05:52:39,426 DEBUG [Thread-2066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:39,426 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,426 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:39,431 INFO [Thread-2070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2070. 2024-11-13T05:52:39,431 DEBUG [Thread-2070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:39,431 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:39,431 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:39,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 416 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477219431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:39,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T05:52:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T05:52:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T05:52:39,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 417 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51610 deadline: 1731477219741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 , the old value is region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=2 because the exception is null or not the one we care about 2024-11-13T05:52:39,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/79f798c526674bddb9a9994f0e8b780c 2024-11-13T05:52:39,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d53d5707282042a68460ce6522d7504a is 50, key is test_row_0/C:col10/1731477158888/Put/seqid=0 2024-11-13T05:52:39,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742373_1549 (size=12301) 2024-11-13T05:52:39,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d53d5707282042a68460ce6522d7504a 2024-11-13T05:52:39,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/4448520cf95044b6bbabe0796b7df27d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/4448520cf95044b6bbabe0796b7df27d 2024-11-13T05:52:39,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/4448520cf95044b6bbabe0796b7df27d, entries=200, sequenceid=518, filesize=14.4 K 2024-11-13T05:52:39,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/79f798c526674bddb9a9994f0e8b780c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/79f798c526674bddb9a9994f0e8b780c 2024-11-13T05:52:39,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/79f798c526674bddb9a9994f0e8b780c, entries=150, sequenceid=518, filesize=12.0 K 2024-11-13T05:52:39,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/d53d5707282042a68460ce6522d7504a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d53d5707282042a68460ce6522d7504a 2024-11-13T05:52:39,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d53d5707282042a68460ce6522d7504a, entries=150, sequenceid=518, filesize=12.0 K 2024-11-13T05:52:39,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for d5b6967321153590da5042dc4609c1e3 in 911ms, sequenceid=518, compaction requested=true 2024-11-13T05:52:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:39,828 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:39,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d5b6967321153590da5042dc4609c1e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:39,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:39,829 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:39,829 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:39,829 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:39,829 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/A is initiating minor compaction (all files) 2024-11-13T05:52:39,830 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/B is initiating minor compaction (all files) 2024-11-13T05:52:39,830 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/B in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,830 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/A in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,830 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a9bed02324a1499e9320a983370d5c96, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9faaa59754e24d388893ae40c6134c81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/79f798c526674bddb9a9994f0e8b780c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.9 K 2024-11-13T05:52:39,830 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c4fca625161b4a74ae85010af2eb73be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c54f5d0acbc44cb4bc79e7ac78ce878e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/4448520cf95044b6bbabe0796b7df27d] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=39.3 K 2024-11-13T05:52:39,830 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a9bed02324a1499e9320a983370d5c96, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731477157332 2024-11-13T05:52:39,830 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4fca625161b4a74ae85010af2eb73be, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731477157332 2024-11-13T05:52:39,830 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9faaa59754e24d388893ae40c6134c81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731477157399 2024-11-13T05:52:39,830 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting c54f5d0acbc44cb4bc79e7ac78ce878e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731477157399 2024-11-13T05:52:39,831 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f798c526674bddb9a9994f0e8b780c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731477158888 2024-11-13T05:52:39,831 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4448520cf95044b6bbabe0796b7df27d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731477158864 2024-11-13T05:52:39,838 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#A#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:39,838 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#B#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:39,838 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/0549fdbe394849eaa2d9748319413be3 is 50, key is test_row_0/A:col10/1731477158888/Put/seqid=0 2024-11-13T05:52:39,838 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/332d972f62f64ea6a6be539f68615b95 is 50, key is test_row_0/B:col10/1731477158888/Put/seqid=0 2024-11-13T05:52:39,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742374_1550 (size=13323) 2024-11-13T05:52:39,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742375_1551 (size=13323) 2024-11-13T05:52:39,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:39,875 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:39,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c6cf59ecbe2e43b8b734a3c872613573 is 50, key is test_row_0/A:col10/1731477159077/Put/seqid=0 2024-11-13T05:52:39,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742376_1552 (size=12301) 2024-11-13T05:52:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T05:52:40,246 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/0549fdbe394849eaa2d9748319413be3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/0549fdbe394849eaa2d9748319413be3 2024-11-13T05:52:40,246 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/332d972f62f64ea6a6be539f68615b95 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/332d972f62f64ea6a6be539f68615b95 2024-11-13T05:52:40,249 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/B of d5b6967321153590da5042dc4609c1e3 into 332d972f62f64ea6a6be539f68615b95(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:40,249 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/A of d5b6967321153590da5042dc4609c1e3 into 0549fdbe394849eaa2d9748319413be3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:40,249 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/A, priority=13, startTime=1731477159828; duration=0sec 2024-11-13T05:52:40,249 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/B, priority=13, startTime=1731477159828; duration=0sec 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:B 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:A 2024-11-13T05:52:40,249 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:40,250 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:40,250 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): d5b6967321153590da5042dc4609c1e3/C is initiating minor compaction (all files) 2024-11-13T05:52:40,250 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d5b6967321153590da5042dc4609c1e3/C in TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:40,250 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ae45c699d344492b837e4332198f4100, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/132db77eb1b1427cad4258fbdc55ee98, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d53d5707282042a68460ce6522d7504a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp, totalSize=36.9 K 2024-11-13T05:52:40,250 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ae45c699d344492b837e4332198f4100, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731477157332 2024-11-13T05:52:40,251 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 132db77eb1b1427cad4258fbdc55ee98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731477157399 2024-11-13T05:52:40,251 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d53d5707282042a68460ce6522d7504a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1731477158888 2024-11-13T05:52:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:40,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. as already flushing 2024-11-13T05:52:40,253 INFO [Thread-2051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2051. 2024-11-13T05:52:40,253 DEBUG [Thread-2051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:40,253 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,253 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:40,256 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d5b6967321153590da5042dc4609c1e3#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:40,257 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/849312b11cde414f918ee492304b2b56 is 50, key is test_row_0/C:col10/1731477158888/Put/seqid=0 2024-11-13T05:52:40,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742377_1553 (size=13323) 2024-11-13T05:52:40,284 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c6cf59ecbe2e43b8b734a3c872613573 2024-11-13T05:52:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a59fb6a9f862451a845193ab1dc9faf6 is 50, key is test_row_0/B:col10/1731477159077/Put/seqid=0 2024-11-13T05:52:40,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742378_1554 (size=12301) 2024-11-13T05:52:40,667 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/849312b11cde414f918ee492304b2b56 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/849312b11cde414f918ee492304b2b56 2024-11-13T05:52:40,672 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d5b6967321153590da5042dc4609c1e3/C of d5b6967321153590da5042dc4609c1e3 into 849312b11cde414f918ee492304b2b56(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:40,672 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:40,672 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., storeName=d5b6967321153590da5042dc4609c1e3/C, priority=13, startTime=1731477159829; duration=0sec 2024-11-13T05:52:40,673 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:40,673 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d5b6967321153590da5042dc4609c1e3:C 2024-11-13T05:52:40,673 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-13T05:52:40,673 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-13T05:52:40,673 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:40,673 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:40,673 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,673 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,673 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,673 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:40,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:40,693 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a59fb6a9f862451a845193ab1dc9faf6 2024-11-13T05:52:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/89c8b6f863504470bd2ca9d5745decc4 is 50, key is test_row_0/C:col10/1731477159077/Put/seqid=0 2024-11-13T05:52:40,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742379_1555 (size=12301) 2024-11-13T05:52:40,713 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-13T05:52:40,713 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:40,713 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,713 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:40,733 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-13T05:52:40,733 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:52:40,733 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,733 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:40,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:41,101 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/89c8b6f863504470bd2ca9d5745decc4 2024-11-13T05:52:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/c6cf59ecbe2e43b8b734a3c872613573 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c6cf59ecbe2e43b8b734a3c872613573 2024-11-13T05:52:41,109 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c6cf59ecbe2e43b8b734a3c872613573, entries=150, sequenceid=531, filesize=12.0 K 2024-11-13T05:52:41,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/a59fb6a9f862451a845193ab1dc9faf6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a59fb6a9f862451a845193ab1dc9faf6 2024-11-13T05:52:41,112 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a59fb6a9f862451a845193ab1dc9faf6, entries=150, sequenceid=531, filesize=12.0 K 2024-11-13T05:52:41,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/89c8b6f863504470bd2ca9d5745decc4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/89c8b6f863504470bd2ca9d5745decc4 2024-11-13T05:52:41,115 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/89c8b6f863504470bd2ca9d5745decc4, entries=150, sequenceid=531, filesize=12.0 K 2024-11-13T05:52:41,116 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=33.54 KB/34350 for d5b6967321153590da5042dc4609c1e3 in 1241ms, sequenceid=531, compaction requested=false 2024-11-13T05:52:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for d5b6967321153590da5042dc4609c1e3: 2024-11-13T05:52:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-13T05:52:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-13T05:52:41,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-13T05:52:41,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0100 sec 2024-11-13T05:52:41,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.0170 sec 2024-11-13T05:52:41,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T05:52:41,252 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 365 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3567 rows 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1190 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3570 rows 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1192 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3575 rows 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1184 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3552 rows 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1196 2024-11-13T05:52:41,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3587 rows 2024-11-13T05:52:41,253 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:52:41,253 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:41,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:41,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:41,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:41,254 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:52:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T05:52:41,257 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477161257"}]},"ts":"1731477161257"} 2024-11-13T05:52:41,258 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:52:41,258 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:52:41,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:52:41,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, UNASSIGN}] 2024-11-13T05:52:41,262 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, UNASSIGN 2024-11-13T05:52:41,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=d5b6967321153590da5042dc4609c1e3, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:41,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, UNASSIGN because future has completed 2024-11-13T05:52:41,264 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:52:41,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:52:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T05:52:41,417 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:41,417 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:52:41,417 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing d5b6967321153590da5042dc4609c1e3, disabling compactions & flushes 2024-11-13T05:52:41,417 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:41,417 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:41,417 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. after waiting 0 ms 2024-11-13T05:52:41,417 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:41,417 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing d5b6967321153590da5042dc4609c1e3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-13T05:52:41,418 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=A 2024-11-13T05:52:41,418 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:41,418 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=B 2024-11-13T05:52:41,418 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:41,418 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d5b6967321153590da5042dc4609c1e3, store=C 2024-11-13T05:52:41,418 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:41,423 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/68f35742c4934128b56949930f2fa8e0 is 50, key is test_row_0/A:col10/1731477160711/Put/seqid=0 2024-11-13T05:52:41,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742380_1556 (size=12301) 2024-11-13T05:52:41,429 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/68f35742c4934128b56949930f2fa8e0 2024-11-13T05:52:41,435 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0939e32ef8ee466d8db83828161c9bc0 is 50, key is test_row_0/B:col10/1731477160711/Put/seqid=0 2024-11-13T05:52:41,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742381_1557 (size=12301) 2024-11-13T05:52:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T05:52:41,839 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0939e32ef8ee466d8db83828161c9bc0 2024-11-13T05:52:41,846 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/167882e06cf3402fa94c1f4f1fe52b7e is 50, key is test_row_0/C:col10/1731477160711/Put/seqid=0 2024-11-13T05:52:41,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742382_1558 (size=12301) 2024-11-13T05:52:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T05:52:42,250 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/167882e06cf3402fa94c1f4f1fe52b7e 2024-11-13T05:52:42,255 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/A/68f35742c4934128b56949930f2fa8e0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/68f35742c4934128b56949930f2fa8e0 2024-11-13T05:52:42,259 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/68f35742c4934128b56949930f2fa8e0, entries=150, sequenceid=542, filesize=12.0 K 2024-11-13T05:52:42,260 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/B/0939e32ef8ee466d8db83828161c9bc0 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0939e32ef8ee466d8db83828161c9bc0 2024-11-13T05:52:42,264 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0939e32ef8ee466d8db83828161c9bc0, entries=150, sequenceid=542, filesize=12.0 K 2024-11-13T05:52:42,265 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/.tmp/C/167882e06cf3402fa94c1f4f1fe52b7e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/167882e06cf3402fa94c1f4f1fe52b7e 2024-11-13T05:52:42,269 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/167882e06cf3402fa94c1f4f1fe52b7e, entries=150, sequenceid=542, filesize=12.0 K 2024-11-13T05:52:42,270 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d5b6967321153590da5042dc4609c1e3 in 853ms, sequenceid=542, compaction requested=true 2024-11-13T05:52:42,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5b2c9b6325be42478a2331b395d827be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/73c5b40a4f5f45dd9e53992796fadc32, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/776c1e02af844462ab0bc84b37c15948, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e457233686ab4a788fa3729ce61fd47b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/675553802af14732b3426b85fdc649d8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/17dd4232b6a34a7dbf6703d6c5622d1e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/30cbfe54ce6341c289d00059d115be9f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d3c0a3e06f95481b81f75d4901dec6d3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/b703e39df7a74f598c2cb87d228df048, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/70c806cc2de64fd9a3d62fbfca4765d0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/98e03574900d447a90c2be868ff7db38, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/8ec0f34059644bc5867d49717e2bcee6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/cffea235d1b54c4188a184544bc877f4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a6cf39e249a4457be34ab62b18cc137, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/3cffbc7390534d688e697a683f79e1d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/60613bd72ed84e2b80eab4659c2ebd2d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/a79d0586a7dd4b618da2711459580b50, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/09701e6baf3245309c68a9dc2b74c7d4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d30915bf776f46f7a11bd76d625d7dfa, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/073084d62dd348a2b089622ebf479516, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e51af00b4521457f9a3c6b212b58ab68, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/2b56b815c9cd41998291c3fee2c1a9d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a0c48514fcc41eda72d3ff8738201c5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/628de2cda2924bd094fd91e6b0b3b8e5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/bdb2bddfa549469e8086d4d56b44b111, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/007e9c55f1ac40d9b8597d3ffd8a8f6b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d7b64965744044cda1de1084dbc1c44c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c4fca625161b4a74ae85010af2eb73be, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c54f5d0acbc44cb4bc79e7ac78ce878e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/4448520cf95044b6bbabe0796b7df27d] to archive 2024-11-13T05:52:42,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:52:42,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5b2c9b6325be42478a2331b395d827be to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5b2c9b6325be42478a2331b395d827be 2024-11-13T05:52:42,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/73c5b40a4f5f45dd9e53992796fadc32 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/73c5b40a4f5f45dd9e53992796fadc32 2024-11-13T05:52:42,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/776c1e02af844462ab0bc84b37c15948 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/776c1e02af844462ab0bc84b37c15948 2024-11-13T05:52:42,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e457233686ab4a788fa3729ce61fd47b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e457233686ab4a788fa3729ce61fd47b 2024-11-13T05:52:42,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/675553802af14732b3426b85fdc649d8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/675553802af14732b3426b85fdc649d8 2024-11-13T05:52:42,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/17dd4232b6a34a7dbf6703d6c5622d1e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/17dd4232b6a34a7dbf6703d6c5622d1e 2024-11-13T05:52:42,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/30cbfe54ce6341c289d00059d115be9f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/30cbfe54ce6341c289d00059d115be9f 2024-11-13T05:52:42,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d3c0a3e06f95481b81f75d4901dec6d3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d3c0a3e06f95481b81f75d4901dec6d3 2024-11-13T05:52:42,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/b703e39df7a74f598c2cb87d228df048 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/b703e39df7a74f598c2cb87d228df048 2024-11-13T05:52:42,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/70c806cc2de64fd9a3d62fbfca4765d0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/70c806cc2de64fd9a3d62fbfca4765d0 2024-11-13T05:52:42,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/98e03574900d447a90c2be868ff7db38 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/98e03574900d447a90c2be868ff7db38 2024-11-13T05:52:42,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/8ec0f34059644bc5867d49717e2bcee6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/8ec0f34059644bc5867d49717e2bcee6 2024-11-13T05:52:42,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/cffea235d1b54c4188a184544bc877f4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/cffea235d1b54c4188a184544bc877f4 2024-11-13T05:52:42,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a6cf39e249a4457be34ab62b18cc137 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a6cf39e249a4457be34ab62b18cc137 2024-11-13T05:52:42,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/3cffbc7390534d688e697a683f79e1d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/3cffbc7390534d688e697a683f79e1d9 2024-11-13T05:52:42,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/60613bd72ed84e2b80eab4659c2ebd2d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/60613bd72ed84e2b80eab4659c2ebd2d 2024-11-13T05:52:42,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/a79d0586a7dd4b618da2711459580b50 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/a79d0586a7dd4b618da2711459580b50 2024-11-13T05:52:42,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/09701e6baf3245309c68a9dc2b74c7d4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/09701e6baf3245309c68a9dc2b74c7d4 2024-11-13T05:52:42,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d30915bf776f46f7a11bd76d625d7dfa to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d30915bf776f46f7a11bd76d625d7dfa 2024-11-13T05:52:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/073084d62dd348a2b089622ebf479516 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/073084d62dd348a2b089622ebf479516 2024-11-13T05:52:42,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e51af00b4521457f9a3c6b212b58ab68 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/e51af00b4521457f9a3c6b212b58ab68 2024-11-13T05:52:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/2b56b815c9cd41998291c3fee2c1a9d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/2b56b815c9cd41998291c3fee2c1a9d9 2024-11-13T05:52:42,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a0c48514fcc41eda72d3ff8738201c5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/5a0c48514fcc41eda72d3ff8738201c5 2024-11-13T05:52:42,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/628de2cda2924bd094fd91e6b0b3b8e5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/628de2cda2924bd094fd91e6b0b3b8e5 2024-11-13T05:52:42,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/bdb2bddfa549469e8086d4d56b44b111 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/bdb2bddfa549469e8086d4d56b44b111 2024-11-13T05:52:42,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/007e9c55f1ac40d9b8597d3ffd8a8f6b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/007e9c55f1ac40d9b8597d3ffd8a8f6b 2024-11-13T05:52:42,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d7b64965744044cda1de1084dbc1c44c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/d7b64965744044cda1de1084dbc1c44c 2024-11-13T05:52:42,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c4fca625161b4a74ae85010af2eb73be to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c4fca625161b4a74ae85010af2eb73be 2024-11-13T05:52:42,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c54f5d0acbc44cb4bc79e7ac78ce878e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c54f5d0acbc44cb4bc79e7ac78ce878e 2024-11-13T05:52:42,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/4448520cf95044b6bbabe0796b7df27d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/4448520cf95044b6bbabe0796b7df27d 2024-11-13T05:52:42,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/39faa31eb07c4e4d80927f342cc2ebe5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/ef1f9f673ce948a28b4cc90f3c6437b5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e8b3d68e57b64005bb79c3c1ed05c568, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/94ea2fad41aa4d63b4d105cafa09901a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/573c22297ab64eaf8750c8e7cc61a189, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e6c160354c8e4329a5e018ff83c10427, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0f052fdbc48a4f988884fe50cee9533c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/c59dadf69e694e278761789d0320d9e1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1544199806ce4bf28960f853c464f243, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9c932ddf4f724e44bbc156489ce98fd1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/30340638f8764cf2bea1315f8a7ad8d9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/d36d19c0b58d4a1ebc3cfc67e707b628, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/b7e225da9ee041e9901dae72f5113d6b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/546e7ad149454a3593ca5a7d7c82da16, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/2125c49307de4168a88bf893cb8805dd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e094378aa4b541449533a9a8f7f2e2c3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/bf24832e5160438084c23e08877b7077, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7023c2736dca4400aa55d06a8c7a8257, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7c62438cced3460cac843f15c4bf7675, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9f7093424adb493191751f06a2bee1ff, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1b4f13826ac54e1a8928db750b56446b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a01b88ba720b45af87129c1b2f8c16ff, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/12a7f509b42f4c9b99f3ec6741319b5d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7f2bbd5d2fa7409db5df08288df2fe3c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/8678934c9c424ac199042dcc3df87c53, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0cb32939432344e98b658a18adabc62c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a9bed02324a1499e9320a983370d5c96, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/14390f8dc28341cbaeafa635ed732dd0, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9faaa59754e24d388893ae40c6134c81, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/79f798c526674bddb9a9994f0e8b780c] to archive 2024-11-13T05:52:42,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:52:42,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/39faa31eb07c4e4d80927f342cc2ebe5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/39faa31eb07c4e4d80927f342cc2ebe5 2024-11-13T05:52:42,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/ef1f9f673ce948a28b4cc90f3c6437b5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/ef1f9f673ce948a28b4cc90f3c6437b5 2024-11-13T05:52:42,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e8b3d68e57b64005bb79c3c1ed05c568 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e8b3d68e57b64005bb79c3c1ed05c568 2024-11-13T05:52:42,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/94ea2fad41aa4d63b4d105cafa09901a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/94ea2fad41aa4d63b4d105cafa09901a 2024-11-13T05:52:42,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/573c22297ab64eaf8750c8e7cc61a189 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/573c22297ab64eaf8750c8e7cc61a189 2024-11-13T05:52:42,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e6c160354c8e4329a5e018ff83c10427 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e6c160354c8e4329a5e018ff83c10427 2024-11-13T05:52:42,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0f052fdbc48a4f988884fe50cee9533c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0f052fdbc48a4f988884fe50cee9533c 2024-11-13T05:52:42,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/c59dadf69e694e278761789d0320d9e1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/c59dadf69e694e278761789d0320d9e1 2024-11-13T05:52:42,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1544199806ce4bf28960f853c464f243 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1544199806ce4bf28960f853c464f243 2024-11-13T05:52:42,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9c932ddf4f724e44bbc156489ce98fd1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9c932ddf4f724e44bbc156489ce98fd1 2024-11-13T05:52:42,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/30340638f8764cf2bea1315f8a7ad8d9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/30340638f8764cf2bea1315f8a7ad8d9 2024-11-13T05:52:42,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/d36d19c0b58d4a1ebc3cfc67e707b628 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/d36d19c0b58d4a1ebc3cfc67e707b628 2024-11-13T05:52:42,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/b7e225da9ee041e9901dae72f5113d6b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/b7e225da9ee041e9901dae72f5113d6b 2024-11-13T05:52:42,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/546e7ad149454a3593ca5a7d7c82da16 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/546e7ad149454a3593ca5a7d7c82da16 2024-11-13T05:52:42,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/2125c49307de4168a88bf893cb8805dd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/2125c49307de4168a88bf893cb8805dd 2024-11-13T05:52:42,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e094378aa4b541449533a9a8f7f2e2c3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/e094378aa4b541449533a9a8f7f2e2c3 2024-11-13T05:52:42,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/bf24832e5160438084c23e08877b7077 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/bf24832e5160438084c23e08877b7077 2024-11-13T05:52:42,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7023c2736dca4400aa55d06a8c7a8257 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7023c2736dca4400aa55d06a8c7a8257 2024-11-13T05:52:42,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7c62438cced3460cac843f15c4bf7675 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7c62438cced3460cac843f15c4bf7675 2024-11-13T05:52:42,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9f7093424adb493191751f06a2bee1ff to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9f7093424adb493191751f06a2bee1ff 2024-11-13T05:52:42,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1b4f13826ac54e1a8928db750b56446b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/1b4f13826ac54e1a8928db750b56446b 2024-11-13T05:52:42,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a01b88ba720b45af87129c1b2f8c16ff to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a01b88ba720b45af87129c1b2f8c16ff 2024-11-13T05:52:42,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/12a7f509b42f4c9b99f3ec6741319b5d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/12a7f509b42f4c9b99f3ec6741319b5d 2024-11-13T05:52:42,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7f2bbd5d2fa7409db5df08288df2fe3c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/7f2bbd5d2fa7409db5df08288df2fe3c 2024-11-13T05:52:42,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/8678934c9c424ac199042dcc3df87c53 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/8678934c9c424ac199042dcc3df87c53 2024-11-13T05:52:42,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0cb32939432344e98b658a18adabc62c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0cb32939432344e98b658a18adabc62c 2024-11-13T05:52:42,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a9bed02324a1499e9320a983370d5c96 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a9bed02324a1499e9320a983370d5c96 2024-11-13T05:52:42,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/14390f8dc28341cbaeafa635ed732dd0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/14390f8dc28341cbaeafa635ed732dd0 2024-11-13T05:52:42,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9faaa59754e24d388893ae40c6134c81 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/9faaa59754e24d388893ae40c6134c81 2024-11-13T05:52:42,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/79f798c526674bddb9a9994f0e8b780c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/79f798c526674bddb9a9994f0e8b780c 2024-11-13T05:52:42,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ddd18ea56183435fb00f1d77cba17b9a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5313a69a250649daa7d77793c1cab218, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2e83c31a51834686bd49a308b97a31db, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2de92af2a13a42beafc6098827a8ae02, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/9ad81f0223d74f5696add122336f0c2a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/3059347c63594231bb3f8723832f8a3a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d329fc057d544c7f879fd3bfdf75a075, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/32e4528ce5484be396e3fd96588c20b9, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/0360f7f6fe7a4bff8c6c2bd5f602127e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/06e062cfb15c4ea0a54ce49018eb13ef, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/4d7b839d1f8f445d8927b3b9ad2d4488, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/adb1f191cb4041fdb6e5469ea7955485, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/cef962fd4465424097d4d7182de070ea, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/37947eec86a0451ab8524bf2ae854f7b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/632e86560a574776b837d5f6450be940, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/8d72e7440dcf40eaad494c45a569fd2c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/fca91299cf784dd98e917523abe76f3c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/118937e1faf9431d90066c964b0a7eae, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/71bbdd6839a64c4185eb330b9a936f90, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d32f2013aba845ab98e60ef98fcacda5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5fea9dd4050a4ba9bbbdab4775784929, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a20374d1fc934b4ba702a1854311c19f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/59e8def5241046329b89d98b61eb7b00, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d67706eeb1e744118d8e1ab01b269a3a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a0225c8098334742a9ef1653b8f20ae7, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ae45c699d344492b837e4332198f4100, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5588e7cd92f8461e944a544271357848, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/132db77eb1b1427cad4258fbdc55ee98, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d53d5707282042a68460ce6522d7504a] to archive 2024-11-13T05:52:42,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:52:42,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ddd18ea56183435fb00f1d77cba17b9a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ddd18ea56183435fb00f1d77cba17b9a 2024-11-13T05:52:42,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5313a69a250649daa7d77793c1cab218 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5313a69a250649daa7d77793c1cab218 2024-11-13T05:52:42,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2e83c31a51834686bd49a308b97a31db to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2e83c31a51834686bd49a308b97a31db 2024-11-13T05:52:42,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2de92af2a13a42beafc6098827a8ae02 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/2de92af2a13a42beafc6098827a8ae02 2024-11-13T05:52:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/9ad81f0223d74f5696add122336f0c2a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/9ad81f0223d74f5696add122336f0c2a 2024-11-13T05:52:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/3059347c63594231bb3f8723832f8a3a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/3059347c63594231bb3f8723832f8a3a 2024-11-13T05:52:42,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d329fc057d544c7f879fd3bfdf75a075 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d329fc057d544c7f879fd3bfdf75a075 2024-11-13T05:52:42,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/32e4528ce5484be396e3fd96588c20b9 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/32e4528ce5484be396e3fd96588c20b9 2024-11-13T05:52:42,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/0360f7f6fe7a4bff8c6c2bd5f602127e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/0360f7f6fe7a4bff8c6c2bd5f602127e 2024-11-13T05:52:42,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/1b60ed3ec2c34cca9ba97d5faf4b2c4b 2024-11-13T05:52:42,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/06e062cfb15c4ea0a54ce49018eb13ef to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/06e062cfb15c4ea0a54ce49018eb13ef 2024-11-13T05:52:42,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/4d7b839d1f8f445d8927b3b9ad2d4488 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/4d7b839d1f8f445d8927b3b9ad2d4488 2024-11-13T05:52:42,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/adb1f191cb4041fdb6e5469ea7955485 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/adb1f191cb4041fdb6e5469ea7955485 2024-11-13T05:52:42,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/cef962fd4465424097d4d7182de070ea to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/cef962fd4465424097d4d7182de070ea 2024-11-13T05:52:42,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/37947eec86a0451ab8524bf2ae854f7b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/37947eec86a0451ab8524bf2ae854f7b 2024-11-13T05:52:42,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/632e86560a574776b837d5f6450be940 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/632e86560a574776b837d5f6450be940 2024-11-13T05:52:42,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/8d72e7440dcf40eaad494c45a569fd2c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/8d72e7440dcf40eaad494c45a569fd2c 2024-11-13T05:52:42,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/fca91299cf784dd98e917523abe76f3c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/fca91299cf784dd98e917523abe76f3c 2024-11-13T05:52:42,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/118937e1faf9431d90066c964b0a7eae to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/118937e1faf9431d90066c964b0a7eae 2024-11-13T05:52:42,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/71bbdd6839a64c4185eb330b9a936f90 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/71bbdd6839a64c4185eb330b9a936f90 2024-11-13T05:52:42,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d32f2013aba845ab98e60ef98fcacda5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d32f2013aba845ab98e60ef98fcacda5 2024-11-13T05:52:42,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5fea9dd4050a4ba9bbbdab4775784929 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5fea9dd4050a4ba9bbbdab4775784929 2024-11-13T05:52:42,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a20374d1fc934b4ba702a1854311c19f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a20374d1fc934b4ba702a1854311c19f 2024-11-13T05:52:42,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/59e8def5241046329b89d98b61eb7b00 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/59e8def5241046329b89d98b61eb7b00 2024-11-13T05:52:42,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d67706eeb1e744118d8e1ab01b269a3a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d67706eeb1e744118d8e1ab01b269a3a 2024-11-13T05:52:42,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a0225c8098334742a9ef1653b8f20ae7 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/a0225c8098334742a9ef1653b8f20ae7 2024-11-13T05:52:42,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ae45c699d344492b837e4332198f4100 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/ae45c699d344492b837e4332198f4100 2024-11-13T05:52:42,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5588e7cd92f8461e944a544271357848 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/5588e7cd92f8461e944a544271357848 2024-11-13T05:52:42,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/132db77eb1b1427cad4258fbdc55ee98 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/132db77eb1b1427cad4258fbdc55ee98 2024-11-13T05:52:42,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d53d5707282042a68460ce6522d7504a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/d53d5707282042a68460ce6522d7504a 2024-11-13T05:52:42,366 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/recovered.edits/545.seqid, newMaxSeqId=545, maxSeqId=1 2024-11-13T05:52:42,367 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. 2024-11-13T05:52:42,368 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for d5b6967321153590da5042dc4609c1e3: Waiting for close lock at 1731477161417Running coprocessor pre-close hooks at 1731477161417Disabling compacts and flushes for region at 1731477161417Disabling writes for close at 1731477161417Obtaining lock to block concurrent updates at 1731477161417Preparing flush snapshotting stores in d5b6967321153590da5042dc4609c1e3 at 1731477161417Finished memstore snapshotting TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731477161418 (+1 ms)Flushing stores of TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3. at 1731477161418Flushing d5b6967321153590da5042dc4609c1e3/A: creating writer at 1731477161418Flushing d5b6967321153590da5042dc4609c1e3/A: appending metadata at 1731477161423 (+5 ms)Flushing d5b6967321153590da5042dc4609c1e3/A: closing flushed file at 1731477161423Flushing d5b6967321153590da5042dc4609c1e3/B: creating writer at 1731477161433 (+10 ms)Flushing d5b6967321153590da5042dc4609c1e3/B: appending metadata at 1731477161435 (+2 ms)Flushing d5b6967321153590da5042dc4609c1e3/B: closing flushed file at 1731477161435Flushing d5b6967321153590da5042dc4609c1e3/C: creating writer at 1731477161843 (+408 ms)Flushing d5b6967321153590da5042dc4609c1e3/C: appending metadata at 1731477161845 (+2 ms)Flushing d5b6967321153590da5042dc4609c1e3/C: closing flushed file at 1731477161845Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10f62960: reopening flushed file at 1731477162254 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b629e68: reopening flushed file at 1731477162259 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d769e5: reopening flushed file at 1731477162264 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d5b6967321153590da5042dc4609c1e3 in 853ms, sequenceid=542, compaction requested=true at 1731477162270 (+6 ms)Writing region close event to WAL at 1731477162364 (+94 ms)Running coprocessor post-close hooks at 1731477162367 (+3 ms)Closed at 1731477162367 2024-11-13T05:52:42,370 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:42,371 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=d5b6967321153590da5042dc4609c1e3, regionState=CLOSED 2024-11-13T05:52:42,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:52:42,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T05:52:42,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure d5b6967321153590da5042dc4609c1e3, server=bc386b7bf2fe,38979,1731477016232 in 1.1100 sec 2024-11-13T05:52:42,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-13T05:52:42,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d5b6967321153590da5042dc4609c1e3, UNASSIGN in 1.1150 sec 2024-11-13T05:52:42,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T05:52:42,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1200 sec 2024-11-13T05:52:42,381 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477162381"}]},"ts":"1731477162381"} 2024-11-13T05:52:42,382 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:52:42,382 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:52:42,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1280 sec 2024-11-13T05:52:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T05:52:42,391 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:42,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:52:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,393 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-13T05:52:42,394 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,397 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:42,399 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/recovered.edits] 2024-11-13T05:52:42,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/0549fdbe394849eaa2d9748319413be3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/0549fdbe394849eaa2d9748319413be3 2024-11-13T05:52:42,403 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/68f35742c4934128b56949930f2fa8e0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/68f35742c4934128b56949930f2fa8e0 2024-11-13T05:52:42,404 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c6cf59ecbe2e43b8b734a3c872613573 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/A/c6cf59ecbe2e43b8b734a3c872613573 2024-11-13T05:52:42,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0939e32ef8ee466d8db83828161c9bc0 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/0939e32ef8ee466d8db83828161c9bc0 2024-11-13T05:52:42,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/332d972f62f64ea6a6be539f68615b95 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/332d972f62f64ea6a6be539f68615b95 2024-11-13T05:52:42,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a59fb6a9f862451a845193ab1dc9faf6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/B/a59fb6a9f862451a845193ab1dc9faf6 2024-11-13T05:52:42,410 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/167882e06cf3402fa94c1f4f1fe52b7e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/167882e06cf3402fa94c1f4f1fe52b7e 2024-11-13T05:52:42,411 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/849312b11cde414f918ee492304b2b56 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/849312b11cde414f918ee492304b2b56 2024-11-13T05:52:42,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/89c8b6f863504470bd2ca9d5745decc4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/C/89c8b6f863504470bd2ca9d5745decc4 2024-11-13T05:52:42,416 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/recovered.edits/545.seqid to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3/recovered.edits/545.seqid 2024-11-13T05:52:42,416 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/d5b6967321153590da5042dc4609c1e3 2024-11-13T05:52:42,416 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:52:42,419 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,423 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:52:42,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:52:42,427 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,427 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:52:42,427 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731477162427"}]},"ts":"9223372036854775807"} 2024-11-13T05:52:42,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:52:42,429 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d5b6967321153590da5042dc4609c1e3, NAME => 'TestAcidGuarantees,,1731477137182.d5b6967321153590da5042dc4609c1e3.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:52:42,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:52:42,430 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731477162429"}]},"ts":"9223372036854775807"} 2024-11-13T05:52:42,432 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:52:42,432 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-13T05:52:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-13T05:52:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:52:42,502 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:42,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=350 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=7620 (was 7294) - AvailableMemoryMB LEAK? - 2024-11-13T05:52:42,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=350, ProcessCount=11, AvailableMemoryMB=7619 2024-11-13T05:52:42,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:52:42,523 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:42,525 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:52:42,526 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:42,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-13T05:52:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:52:42,527 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:52:42,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742383_1559 (size=1061) 2024-11-13T05:52:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:52:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:52:42,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468 2024-11-13T05:52:42,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742384_1560 (size=53) 2024-11-13T05:52:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:52:43,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:43,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 636fe54399d3e25dd08446a6e1c82434, disabling compactions & flushes 2024-11-13T05:52:43,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. after waiting 0 ms 2024-11-13T05:52:43,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 636fe54399d3e25dd08446a6e1c82434: Waiting for close lock at 1731477163342Disabling compacts and flushes for region at 1731477163342Disabling writes for close at 1731477163342Writing region close event to WAL at 1731477163342Closed at 1731477163342 2024-11-13T05:52:43,343 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:52:43,343 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731477163343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477163343"}]},"ts":"1731477163343"} 2024-11-13T05:52:43,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:52:43,346 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:52:43,346 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477163346"}]},"ts":"1731477163346"} 2024-11-13T05:52:43,348 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T05:52:43,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, ASSIGN}] 2024-11-13T05:52:43,349 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, ASSIGN 2024-11-13T05:52:43,350 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=false 2024-11-13T05:52:43,501 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:43,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, ASSIGN because future has completed 2024-11-13T05:52:43,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:52:43,658 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:43,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:43,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,659 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,659 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,661 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:43,661 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 636fe54399d3e25dd08446a6e1c82434 columnFamilyName A 2024-11-13T05:52:43,661 DEBUG [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:43,661 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(327): Store=636fe54399d3e25dd08446a6e1c82434/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:43,662 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:52:43,663 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:43,663 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 636fe54399d3e25dd08446a6e1c82434 columnFamilyName B 2024-11-13T05:52:43,663 DEBUG [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:43,663 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(327): Store=636fe54399d3e25dd08446a6e1c82434/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:43,663 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,664 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:43,664 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 636fe54399d3e25dd08446a6e1c82434 columnFamilyName C 2024-11-13T05:52:43,664 DEBUG [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:43,665 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(327): Store=636fe54399d3e25dd08446a6e1c82434/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:43,665 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,665 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,666 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,666 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,667 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,667 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,667 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:52:43,668 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:43,670 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 636fe54399d3e25dd08446a6e1c82434; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66928397, jitterRate=-0.002689167857170105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:52:43,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:43,671 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 636fe54399d3e25dd08446a6e1c82434: Running coprocessor pre-open hook at 1731477163659Writing region info on filesystem at 1731477163659Initializing all the Stores at 1731477163659Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477163659Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477163659Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477163659Cleaning up temporary data from old regions at 1731477163667 (+8 ms)Running coprocessor post-open hooks at 1731477163670 (+3 ms)Region opened successfully at 1731477163671 (+1 ms) 2024-11-13T05:52:43,672 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., pid=146, masterSystemTime=1731477163655 2024-11-13T05:52:43,674 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,674 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:43,675 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:43,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:52:43,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-13T05:52:43,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 in 174 msec 2024-11-13T05:52:43,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-13T05:52:43,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, ASSIGN in 331 msec 2024-11-13T05:52:43,681 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:52:43,682 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477163682"}]},"ts":"1731477163682"} 2024-11-13T05:52:43,683 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T05:52:43,684 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:52:43,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-13T05:52:44,496 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:52:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-13T05:52:44,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:44,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:44,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:44,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:44,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8e031c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:44,692 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c41e6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:44,694 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:44,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:44,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:52:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,44267,1731477015408 2024-11-13T05:52:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bd5f386 2024-11-13T05:52:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:52:44,702 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:52:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T05:52:44,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T05:52:44,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742385_1561 (size=1097) 2024-11-13T05:52:45,148 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-13T05:52:45,148 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-13T05:52:45,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:52:45,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, REOPEN/MOVE}] 2024-11-13T05:52:45,163 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, REOPEN/MOVE 2024-11-13T05:52:45,164 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:45,167 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44267 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=bc386b7bf2fe,38979,1731477016232, table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T05:52:45,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, REOPEN/MOVE because future has completed 2024-11-13T05:52:45,168 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:52:45,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:52:45,323 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,323 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:52:45,323 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 636fe54399d3e25dd08446a6e1c82434, disabling compactions & flushes 2024-11-13T05:52:45,323 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,323 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,323 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. after waiting 0 ms 2024-11-13T05:52:45,323 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,337 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:52:45,338 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,338 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 636fe54399d3e25dd08446a6e1c82434: Waiting for close lock at 1731477165323Running coprocessor pre-close hooks at 1731477165323Disabling compacts and flushes for region at 1731477165323Disabling writes for close at 1731477165323Writing region close event to WAL at 1731477165335 (+12 ms)Running coprocessor post-close hooks at 1731477165338 (+3 ms)Closed at 1731477165338 2024-11-13T05:52:45,338 WARN [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 636fe54399d3e25dd08446a6e1c82434 to self. 2024-11-13T05:52:45,340 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=CLOSED 2024-11-13T05:52:45,341 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:52:45,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-13T05:52:45,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 in 175 msec 2024-11-13T05:52:45,349 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,38979,1731477016232; forceNewPlan=false, retain=true 2024-11-13T05:52:45,500 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=OPENING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:45,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, REOPEN/MOVE because future has completed 2024-11-13T05:52:45,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:52:45,657 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,657 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:45,657 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,657 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:45,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,668 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,669 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:45,669 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 636fe54399d3e25dd08446a6e1c82434 columnFamilyName A 2024-11-13T05:52:45,670 DEBUG [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:45,671 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(327): Store=636fe54399d3e25dd08446a6e1c82434/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:45,671 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,672 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:45,672 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 636fe54399d3e25dd08446a6e1c82434 columnFamilyName B 2024-11-13T05:52:45,672 DEBUG [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:45,672 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(327): Store=636fe54399d3e25dd08446a6e1c82434/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:45,672 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,673 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-13T05:52:45,673 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 636fe54399d3e25dd08446a6e1c82434 columnFamilyName C 2024-11-13T05:52:45,673 DEBUG [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:45,673 INFO [StoreOpener-636fe54399d3e25dd08446a6e1c82434-1 {}] regionserver.HStore(327): Store=636fe54399d3e25dd08446a6e1c82434/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:45,674 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,674 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,674 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,675 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,679 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,679 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,679 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:52:45,681 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,682 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 636fe54399d3e25dd08446a6e1c82434; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72910173, jitterRate=0.08644624054431915}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:52:45,682 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:45,683 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 636fe54399d3e25dd08446a6e1c82434: Running coprocessor pre-open hook at 1731477165658Writing region info on filesystem at 1731477165658Initializing all the Stores at 1731477165665 (+7 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477165665Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477165668 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477165668Cleaning up temporary data from old regions at 1731477165679 (+11 ms)Running coprocessor post-open hooks at 1731477165682 (+3 ms)Region opened successfully at 1731477165683 (+1 ms) 2024-11-13T05:52:45,687 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., pid=151, masterSystemTime=1731477165653 2024-11-13T05:52:45,691 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,691 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:45,692 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:45,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:52:45,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-13T05:52:45,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 in 194 msec 2024-11-13T05:52:45,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T05:52:45,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, REOPEN/MOVE in 537 msec 2024-11-13T05:52:45,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-13T05:52:45,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 549 msec 2024-11-13T05:52:45,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0040 sec 2024-11-13T05:52:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-13T05:52:45,719 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:45,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6eabbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b118012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a3d36b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3be2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,784 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f376e36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,786 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f633599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,861 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640680d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,862 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d4055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d289710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f18b7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23975ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,872 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18eedf14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162f1ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,876 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fdc8af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d13a7c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e62e123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,917 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476aa6f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c7c01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b72907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:45,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:45,966 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e87f393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:45,967 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72223a45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1043a3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:46,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:46,021 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:46,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6f5a11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55fcd32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,44267,-1 for getting cluster id 2024-11-13T05:52:46,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:46,028 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f8f3e79-255c-44e7-af82-b12e7afebc35' 2024-11-13T05:52:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f8f3e79-255c-44e7-af82-b12e7afebc35" 2024-11-13T05:52:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa95ec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,44267,-1] 2024-11-13T05:52:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:46,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a077dd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:46,056 DEBUG [Thread-2500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,057 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,057 DEBUG [Thread-2506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,057 DEBUG [Thread-2502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,059 DEBUG [Thread-2504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,060 DEBUG [Thread-2504 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,086 DEBUG [Thread-2504 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,086 DEBUG [Thread-2487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,087 DEBUG [Thread-2489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,088 DEBUG [Thread-2493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,091 DEBUG [Thread-2495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:46,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:52:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:46,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:46,117 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:46,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38979,1731477016232, seqNum=-1] 2024-11-13T05:52:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:46,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5] 2024-11-13T05:52:46,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T05:52:46,158 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T05:52:46,159 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:46,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:46,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477166165, max timestamp=1731477166177Num uniques 859; 2024-11-13T05:52:46,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477166165, max timestamp=1731477166177Num uniques 859; 2024-11-13T05:52:46,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:46,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:46,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477166165, max timestamp=1731477166177Num uniques 859; 2024-11-13T05:52:46,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:46,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bdbc57c6f6f24554afc4836b55f067e8_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477166086/Put/seqid=0 2024-11-13T05:52:46,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742386_1562 (size=12154) 2024-11-13T05:52:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477226194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477226195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477226197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477226198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477226198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T05:52:46,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477226301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477226312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477226312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477226312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477226312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T05:52:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477226511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477226522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477226522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477226522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477226522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:46,593 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bdbc57c6f6f24554afc4836b55f067e8_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bdbc57c6f6f24554afc4836b55f067e8_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:46,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4735a1d6aa064be482f681c9326db7b4, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:46,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4735a1d6aa064be482f681c9326db7b4 is 175, key is test_row_0/A:col10/1731477166086/Put/seqid=0 2024-11-13T05:52:46,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742387_1563 (size=30955) 2024-11-13T05:52:46,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:46,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:46,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T05:52:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477226821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477226841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477226842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477226842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477226842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:46,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:46,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:46,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:46,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:46,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4735a1d6aa064be482f681c9326db7b4 2024-11-13T05:52:47,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/fd24063feae646bdb692994e85272993 is 50, key is test_row_0/B:col10/1731477166086/Put/seqid=0 2024-11-13T05:52:47,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742388_1564 (size=12001) 2024-11-13T05:52:47,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/fd24063feae646bdb692994e85272993 2024-11-13T05:52:47,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/4ebaf5b263ce4492abbc6b8dfe5e9532 is 50, key is test_row_0/C:col10/1731477166086/Put/seqid=0 2024-11-13T05:52:47,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742389_1565 (size=12001) 2024-11-13T05:52:47,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/4ebaf5b263ce4492abbc6b8dfe5e9532 2024-11-13T05:52:47,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:47,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:47,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:47,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4735a1d6aa064be482f681c9326db7b4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4 2024-11-13T05:52:47,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4, entries=150, sequenceid=14, filesize=30.2 K 2024-11-13T05:52:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/fd24063feae646bdb692994e85272993 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/fd24063feae646bdb692994e85272993 2024-11-13T05:52:47,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/fd24063feae646bdb692994e85272993, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:52:47,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/4ebaf5b263ce4492abbc6b8dfe5e9532 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/4ebaf5b263ce4492abbc6b8dfe5e9532 2024-11-13T05:52:47,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/4ebaf5b263ce4492abbc6b8dfe5e9532, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T05:52:47,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 636fe54399d3e25dd08446a6e1c82434 in 998ms, sequenceid=14, compaction requested=false 2024-11-13T05:52:47,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:47,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T05:52:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:47,230 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-13T05:52:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:47,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:47,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111383cd3dc9a8d5474faaf74a84dad93478_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477166192/Put/seqid=0 2024-11-13T05:52:47,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742390_1566 (size=12154) 2024-11-13T05:52:47,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:47,255 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111383cd3dc9a8d5474faaf74a84dad93478_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383cd3dc9a8d5474faaf74a84dad93478_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/f05a065784404a60a0c28a9869a7ac1c, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:47,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/f05a065784404a60a0c28a9869a7ac1c is 175, key is test_row_0/A:col10/1731477166192/Put/seqid=0 2024-11-13T05:52:47,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742391_1567 (size=30955) 2024-11-13T05:52:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T05:52:47,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477227351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477227353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477227353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477227356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477227358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477227461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477227471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477227471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,669 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/f05a065784404a60a0c28a9869a7ac1c 2024-11-13T05:52:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477227671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/5671c70bff8346d8a8e4330257cc819d is 50, key is test_row_0/B:col10/1731477166192/Put/seqid=0 2024-11-13T05:52:47,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742392_1568 (size=12001) 2024-11-13T05:52:47,679 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/5671c70bff8346d8a8e4330257cc819d 2024-11-13T05:52:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/5ef7b7b3b664427ba04e02552c0c61df is 50, key is test_row_0/C:col10/1731477166192/Put/seqid=0 2024-11-13T05:52:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477227681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477227681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742393_1569 (size=12001) 2024-11-13T05:52:47,892 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:52:47,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477227981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477227992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477227992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,088 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/5ef7b7b3b664427ba04e02552c0c61df 2024-11-13T05:52:48,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/f05a065784404a60a0c28a9869a7ac1c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c 2024-11-13T05:52:48,094 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c, entries=150, sequenceid=51, filesize=30.2 K 2024-11-13T05:52:48,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/5671c70bff8346d8a8e4330257cc819d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5671c70bff8346d8a8e4330257cc819d 2024-11-13T05:52:48,097 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5671c70bff8346d8a8e4330257cc819d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T05:52:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/5ef7b7b3b664427ba04e02552c0c61df as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/5ef7b7b3b664427ba04e02552c0c61df 2024-11-13T05:52:48,100 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/5ef7b7b3b664427ba04e02552c0c61df, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T05:52:48,100 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 636fe54399d3e25dd08446a6e1c82434 in 870ms, sequenceid=51, compaction requested=false 2024-11-13T05:52:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T05:52:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T05:52:48,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T05:52:48,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9420 sec 2024-11-13T05:52:48,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.9480 sec 2024-11-13T05:52:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T05:52:48,311 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:48,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T05:52:48,313 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:48,314 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:48,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:48,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T05:52:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:48,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b0141e9c882b4eaa8efb6945359f9460_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:48,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742394_1570 (size=12154) 2024-11-13T05:52:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477168394, max timestamp=1731477168422Num uniques 859; 2024-11-13T05:52:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477168394, max timestamp=1731477168422Num uniques 859; 2024-11-13T05:52:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477168394, max timestamp=1731477168422Num uniques 859; 2024-11-13T05:52:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:48,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:48,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477228488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477228491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477228496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477228501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477228512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477228601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477228611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:48,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:48,804 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b0141e9c882b4eaa8efb6945359f9460_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b0141e9c882b4eaa8efb6945359f9460_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:48,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9f7d2c67ab374ea4bfc63dfa402bbdf3, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:48,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9f7d2c67ab374ea4bfc63dfa402bbdf3 is 175, key is test_row_0/A:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:48,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742395_1571 (size=30955) 2024-11-13T05:52:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477228811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477228822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:48,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:48,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:49,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477229121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477229132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9f7d2c67ab374ea4bfc63dfa402bbdf3 2024-11-13T05:52:49,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/ec69044376d14e38b5f7e5af4b31a0d6 is 50, key is test_row_0/B:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:49,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742396_1572 (size=12001) 2024-11-13T05:52:49,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:49,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477229512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477229512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,515 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:49,515 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477229531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,535 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/ec69044376d14e38b5f7e5af4b31a0d6 2024-11-13T05:52:49,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/83d8ced5556a493b8bae57d622d30e14 is 50, key is test_row_0/C:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:49,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742397_1573 (size=12001) 2024-11-13T05:52:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477229631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477229641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:49,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:49,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:50,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/83d8ced5556a493b8bae57d622d30e14 2024-11-13T05:52:50,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9f7d2c67ab374ea4bfc63dfa402bbdf3 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3 2024-11-13T05:52:50,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3, entries=150, sequenceid=63, filesize=30.2 K 2024-11-13T05:52:50,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/ec69044376d14e38b5f7e5af4b31a0d6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec69044376d14e38b5f7e5af4b31a0d6 2024-11-13T05:52:50,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec69044376d14e38b5f7e5af4b31a0d6, entries=150, sequenceid=63, filesize=11.7 K 2024-11-13T05:52:50,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/83d8ced5556a493b8bae57d622d30e14 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/83d8ced5556a493b8bae57d622d30e14 2024-11-13T05:52:50,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/83d8ced5556a493b8bae57d622d30e14, entries=150, sequenceid=63, filesize=11.7 K 2024-11-13T05:52:50,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 636fe54399d3e25dd08446a6e1c82434 in 1677ms, sequenceid=63, compaction requested=true 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:50,051 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:50,051 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:50,052 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:50,052 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:52:50,052 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:50,052 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/fd24063feae646bdb692994e85272993, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5671c70bff8346d8a8e4330257cc819d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec69044376d14e38b5f7e5af4b31a0d6] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.2 K 2024-11-13T05:52:50,052 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:50,053 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:52:50,053 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:50,053 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=90.7 K 2024-11-13T05:52:50,053 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:50,053 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3] 2024-11-13T05:52:50,053 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting fd24063feae646bdb692994e85272993, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477166075 2024-11-13T05:52:50,053 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4735a1d6aa064be482f681c9326db7b4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477166075 2024-11-13T05:52:50,053 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5671c70bff8346d8a8e4330257cc819d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731477166192 2024-11-13T05:52:50,053 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05a065784404a60a0c28a9869a7ac1c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731477166192 2024-11-13T05:52:50,054 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7d2c67ab374ea4bfc63dfa402bbdf3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731477167340 2024-11-13T05:52:50,054 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ec69044376d14e38b5f7e5af4b31a0d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731477167340 2024-11-13T05:52:50,064 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:50,065 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:50,065 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/11f0fb29b3ca4ce8bdc73d7d05f88a87 is 50, key is test_row_0/B:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:50,066 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113243356d477f543758ba74230510b2b17_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:50,068 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113243356d477f543758ba74230510b2b17_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:50,068 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113243356d477f543758ba74230510b2b17_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:50,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742398_1574 (size=12104) 2024-11-13T05:52:50,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742399_1575 (size=4469) 2024-11-13T05:52:50,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T05:52:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:50,146 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-13T05:52:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:50,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f044bda47ee44f3a8a11741c35bdc20_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477168493/Put/seqid=0 2024-11-13T05:52:50,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742400_1576 (size=12154) 2024-11-13T05:52:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:50,476 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/11f0fb29b3ca4ce8bdc73d7d05f88a87 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/11f0fb29b3ca4ce8bdc73d7d05f88a87 2024-11-13T05:52:50,483 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into 11f0fb29b3ca4ce8bdc73d7d05f88a87(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:50,483 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:50,483 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477170051; duration=0sec 2024-11-13T05:52:50,483 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:50,483 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:52:50,483 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:50,484 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:50,484 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:52:50,484 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:50,484 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/4ebaf5b263ce4492abbc6b8dfe5e9532, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/5ef7b7b3b664427ba04e02552c0c61df, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/83d8ced5556a493b8bae57d622d30e14] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.2 K 2024-11-13T05:52:50,486 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ebaf5b263ce4492abbc6b8dfe5e9532, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731477166075 2024-11-13T05:52:50,486 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:50,486 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef7b7b3b664427ba04e02552c0c61df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731477166192 2024-11-13T05:52:50,486 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/87cb7745d1c6484a9c08f82df744830c is 175, key is test_row_0/A:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:50,486 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d8ced5556a493b8bae57d622d30e14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731477167340 2024-11-13T05:52:50,499 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:50,500 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/074da16739ea4c46a12fdf9c98ab0184 is 50, key is test_row_0/C:col10/1731477168372/Put/seqid=0 2024-11-13T05:52:50,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742401_1577 (size=31058) 2024-11-13T05:52:50,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742402_1578 (size=12104) 2024-11-13T05:52:50,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:50,560 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f044bda47ee44f3a8a11741c35bdc20_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f044bda47ee44f3a8a11741c35bdc20_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:50,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5d8d74e4b1fe4dceb1abd9b4d485b862, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:50,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5d8d74e4b1fe4dceb1abd9b4d485b862 is 175, key is test_row_0/A:col10/1731477168493/Put/seqid=0 2024-11-13T05:52:50,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742403_1579 (size=30955) 2024-11-13T05:52:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:50,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477230672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:50,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477230678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:50,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:50,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477230782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:50,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477230791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:50,905 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/87cb7745d1c6484a9c08f82df744830c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/87cb7745d1c6484a9c08f82df744830c 2024-11-13T05:52:50,910 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into 87cb7745d1c6484a9c08f82df744830c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:50,910 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:50,910 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477170051; duration=0sec 2024-11-13T05:52:50,910 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:50,910 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:52:50,938 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/074da16739ea4c46a12fdf9c98ab0184 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/074da16739ea4c46a12fdf9c98ab0184 2024-11-13T05:52:50,949 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into 074da16739ea4c46a12fdf9c98ab0184(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:50,949 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:50,949 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477170051; duration=0sec 2024-11-13T05:52:50,949 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:50,949 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:52:50,992 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5d8d74e4b1fe4dceb1abd9b4d485b862 2024-11-13T05:52:50,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477230992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477231002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/e56dd0a2a86045f0b3f28b0bbc1ef33d is 50, key is test_row_0/B:col10/1731477168493/Put/seqid=0 2024-11-13T05:52:51,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742404_1580 (size=12001) 2024-11-13T05:52:51,037 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/e56dd0a2a86045f0b3f28b0bbc1ef33d 2024-11-13T05:52:51,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/336dc73504c34e338e4d43897cedd235 is 50, key is test_row_0/C:col10/1731477168493/Put/seqid=0 2024-11-13T05:52:51,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742405_1581 (size=12001) 2024-11-13T05:52:51,060 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/336dc73504c34e338e4d43897cedd235 2024-11-13T05:52:51,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5d8d74e4b1fe4dceb1abd9b4d485b862 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862 2024-11-13T05:52:51,066 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862, entries=150, sequenceid=97, filesize=30.2 K 2024-11-13T05:52:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/e56dd0a2a86045f0b3f28b0bbc1ef33d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e56dd0a2a86045f0b3f28b0bbc1ef33d 2024-11-13T05:52:51,069 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e56dd0a2a86045f0b3f28b0bbc1ef33d, entries=150, sequenceid=97, filesize=11.7 K 2024-11-13T05:52:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/336dc73504c34e338e4d43897cedd235 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/336dc73504c34e338e4d43897cedd235 2024-11-13T05:52:51,074 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/336dc73504c34e338e4d43897cedd235, entries=150, sequenceid=97, filesize=11.7 K 2024-11-13T05:52:51,075 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 636fe54399d3e25dd08446a6e1c82434 in 929ms, sequenceid=97, compaction requested=false 2024-11-13T05:52:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T05:52:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T05:52:51,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T05:52:51,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7630 sec 2024-11-13T05:52:51,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7660 sec 2024-11-13T05:52:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:51,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:52:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:51,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:51,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366ec59a01fa14f488be3a7dc5ce255f4_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:51,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742406_1582 (size=14594) 2024-11-13T05:52:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477231344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477231348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477231451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477231461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477231521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,524 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_1' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477231531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,533 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_2' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477231541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,545 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477231661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477231671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,712 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:51,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111366ec59a01fa14f488be3a7dc5ce255f4_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366ec59a01fa14f488be3a7dc5ce255f4_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:51,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8398e27574284ea1912649375018a4b8, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:51,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8398e27574284ea1912649375018a4b8 is 175, key is test_row_0/A:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:51,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742407_1583 (size=39549) 2024-11-13T05:52:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477231971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:51,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477231981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:52,120 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8398e27574284ea1912649375018a4b8 2024-11-13T05:52:52,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/5cd76bfdc72e4d4d899085d65c7251eb is 50, key is test_row_0/B:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:52,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742408_1584 (size=12001) 2024-11-13T05:52:52,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/5cd76bfdc72e4d4d899085d65c7251eb 2024-11-13T05:52:52,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/9d9f4174719849e883383aa034a31705 is 50, key is test_row_0/C:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:52,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742409_1585 (size=12001) 2024-11-13T05:52:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T05:52:52,472 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:52,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T05:52:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T05:52:52,474 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:52,474 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:52,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:52,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477232482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477232491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:52,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:52,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/9d9f4174719849e883383aa034a31705 2024-11-13T05:52:52,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8398e27574284ea1912649375018a4b8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8 2024-11-13T05:52:52,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8, entries=200, sequenceid=113, filesize=38.6 K 2024-11-13T05:52:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/5cd76bfdc72e4d4d899085d65c7251eb as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5cd76bfdc72e4d4d899085d65c7251eb 2024-11-13T05:52:52,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5cd76bfdc72e4d4d899085d65c7251eb, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T05:52:52,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/9d9f4174719849e883383aa034a31705 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9d9f4174719849e883383aa034a31705 2024-11-13T05:52:52,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9d9f4174719849e883383aa034a31705, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T05:52:52,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 636fe54399d3e25dd08446a6e1c82434 in 1247ms, sequenceid=113, compaction requested=true 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:52,552 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:52,552 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:52,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:52,552 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:52,552 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:52,552 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:52:52,552 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:52:52,552 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:52,552 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:52,553 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/11f0fb29b3ca4ce8bdc73d7d05f88a87, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e56dd0a2a86045f0b3f28b0bbc1ef33d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5cd76bfdc72e4d4d899085d65c7251eb] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.3 K 2024-11-13T05:52:52,553 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/87cb7745d1c6484a9c08f82df744830c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=99.2 K 2024-11-13T05:52:52,553 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/87cb7745d1c6484a9c08f82df744830c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8] 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f0fb29b3ca4ce8bdc73d7d05f88a87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731477167340 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87cb7745d1c6484a9c08f82df744830c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731477167340 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e56dd0a2a86045f0b3f28b0bbc1ef33d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731477168476 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d8d74e4b1fe4dceb1abd9b4d485b862, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731477168476 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd76bfdc72e4d4d899085d65c7251eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731477170660 2024-11-13T05:52:52,553 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8398e27574284ea1912649375018a4b8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731477170660 2024-11-13T05:52:52,557 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:52,558 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111390cd208765264359a4aea5a583cce309_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:52,559 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:52,559 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/2e54af0e23894c4b989e6618322acb4b is 50, key is test_row_0/B:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:52,560 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111390cd208765264359a4aea5a583cce309_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:52,560 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111390cd208765264359a4aea5a583cce309_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:52,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742410_1586 (size=12207) 2024-11-13T05:52:52,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742411_1587 (size=4469) 2024-11-13T05:52:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T05:52:52,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T05:52:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:52,627 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T05:52:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e3aaa85ba6d45dc8fb25bf986ee59e2_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477171337/Put/seqid=0 2024-11-13T05:52:52,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742412_1588 (size=12254) 2024-11-13T05:52:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T05:52:52,964 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:52,964 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/44335731470f4599b1475e1476bf6024 is 175, key is test_row_0/A:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:52,967 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/2e54af0e23894c4b989e6618322acb4b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/2e54af0e23894c4b989e6618322acb4b 2024-11-13T05:52:52,971 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into 2e54af0e23894c4b989e6618322acb4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:52,971 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:52,971 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477172552; duration=0sec 2024-11-13T05:52:52,972 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:52,972 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:52:52,972 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:52,972 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:52,972 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:52:52,972 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:52,973 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/074da16739ea4c46a12fdf9c98ab0184, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/336dc73504c34e338e4d43897cedd235, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9d9f4174719849e883383aa034a31705] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.3 K 2024-11-13T05:52:52,973 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 074da16739ea4c46a12fdf9c98ab0184, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731477167340 2024-11-13T05:52:52,973 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 336dc73504c34e338e4d43897cedd235, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731477168476 2024-11-13T05:52:52,973 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9f4174719849e883383aa034a31705, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731477170660 2024-11-13T05:52:52,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742413_1589 (size=31161) 2024-11-13T05:52:52,980 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:52,981 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/de3fb15113be474f8a5011e60f3bcffe is 50, key is test_row_0/C:col10/1731477170660/Put/seqid=0 2024-11-13T05:52:52,983 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/44335731470f4599b1475e1476bf6024 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/44335731470f4599b1475e1476bf6024 2024-11-13T05:52:52,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742414_1590 (size=12207) 2024-11-13T05:52:52,991 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/de3fb15113be474f8a5011e60f3bcffe as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/de3fb15113be474f8a5011e60f3bcffe 2024-11-13T05:52:52,991 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into 44335731470f4599b1475e1476bf6024(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:52,991 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:52,992 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477172552; duration=0sec 2024-11-13T05:52:52,992 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:52,992 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:52:52,995 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into de3fb15113be474f8a5011e60f3bcffe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:52,995 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:52,995 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477172552; duration=0sec 2024-11-13T05:52:52,996 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:52,996 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:52:53,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:53,040 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e3aaa85ba6d45dc8fb25bf986ee59e2_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e3aaa85ba6d45dc8fb25bf986ee59e2_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5ac815f15e94495db955ac3be922e603, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5ac815f15e94495db955ac3be922e603 is 175, key is test_row_0/A:col10/1731477171337/Put/seqid=0 2024-11-13T05:52:53,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742415_1591 (size=31055) 2024-11-13T05:52:53,050 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5ac815f15e94495db955ac3be922e603 2024-11-13T05:52:53,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/aa581a094fc44faeb62ad496d7562747 is 50, key is test_row_0/B:col10/1731477171337/Put/seqid=0 2024-11-13T05:52:53,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742416_1592 (size=12101) 2024-11-13T05:52:53,085 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/aa581a094fc44faeb62ad496d7562747 2024-11-13T05:52:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/a05f8aa1e607465f959f788744920a20 is 50, key is test_row_0/C:col10/1731477171337/Put/seqid=0 2024-11-13T05:52:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T05:52:53,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742417_1593 (size=12101) 2024-11-13T05:52:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:53,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:53,511 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/a05f8aa1e607465f959f788744920a20 2024-11-13T05:52:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/5ac815f15e94495db955ac3be922e603 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603 2024-11-13T05:52:53,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477233516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,520 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603, entries=150, sequenceid=132, filesize=30.3 K 2024-11-13T05:52:53,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/aa581a094fc44faeb62ad496d7562747 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/aa581a094fc44faeb62ad496d7562747 2024-11-13T05:52:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477233518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,524 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/aa581a094fc44faeb62ad496d7562747, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T05:52:53,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/a05f8aa1e607465f959f788744920a20 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/a05f8aa1e607465f959f788744920a20 2024-11-13T05:52:53,528 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/a05f8aa1e607465f959f788744920a20, entries=150, sequenceid=132, filesize=11.8 K 2024-11-13T05:52:53,529 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 636fe54399d3e25dd08446a6e1c82434 in 903ms, sequenceid=132, compaction requested=false 2024-11-13T05:52:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T05:52:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T05:52:53,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T05:52:53,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0560 sec 2024-11-13T05:52:53,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0600 sec 2024-11-13T05:52:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T05:52:53,611 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:53,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T05:52:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T05:52:53,614 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:53,615 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:53,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:53,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-13T05:52:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:53,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:53,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111374569f2a61ac4d21af6f68c7cd085492_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:53,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742418_1594 (size=14794) 2024-11-13T05:52:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477233674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477233674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T05:52:53,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:53,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:53,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:53,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:53,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:53,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477233782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477233782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:53,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:53,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:53,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T05:52:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477233991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477233992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:54,052 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:54,056 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111374569f2a61ac4d21af6f68c7cd085492_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111374569f2a61ac4d21af6f68c7cd085492_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:54,057 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/fecd5cc89f3949f9986817b49305ae0c, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:54,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/fecd5cc89f3949f9986817b49305ae0c is 175, key is test_row_0/A:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:54,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742419_1595 (size=39749) 2024-11-13T05:52:54,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T05:52:54,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477234302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:54,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477234302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:54,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/fecd5cc89f3949f9986817b49305ae0c 2024-11-13T05:52:54,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/f559d2e24f8e486785b0fcd6e5a01c8f is 50, key is test_row_0/B:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:54,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742420_1596 (size=12151) 2024-11-13T05:52:54,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T05:52:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477234811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477234811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:54,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:54,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/f559d2e24f8e486785b0fcd6e5a01c8f 2024-11-13T05:52:54,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/27d5fd6121fe46d8b352aac7b2785e20 is 50, key is test_row_0/C:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:54,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742421_1597 (size=12151) 2024-11-13T05:52:54,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/27d5fd6121fe46d8b352aac7b2785e20 2024-11-13T05:52:54,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/fecd5cc89f3949f9986817b49305ae0c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c 2024-11-13T05:52:54,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c, entries=200, sequenceid=149, filesize=38.8 K 2024-11-13T05:52:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/f559d2e24f8e486785b0fcd6e5a01c8f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/f559d2e24f8e486785b0fcd6e5a01c8f 2024-11-13T05:52:54,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/f559d2e24f8e486785b0fcd6e5a01c8f, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T05:52:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/27d5fd6121fe46d8b352aac7b2785e20 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/27d5fd6121fe46d8b352aac7b2785e20 2024-11-13T05:52:54,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/27d5fd6121fe46d8b352aac7b2785e20, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T05:52:54,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 636fe54399d3e25dd08446a6e1c82434 in 1365ms, sequenceid=149, compaction requested=true 2024-11-13T05:52:54,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:54,988 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:54,989 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:54,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:54,989 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:54,990 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:52:54,990 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,990 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/44335731470f4599b1475e1476bf6024, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=99.6 K 2024-11-13T05:52:54,990 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,990 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/44335731470f4599b1475e1476bf6024, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c] 2024-11-13T05:52:54,990 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44335731470f4599b1475e1476bf6024, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731477170660 2024-11-13T05:52:54,991 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:54,991 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac815f15e94495db955ac3be922e603, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731477171337 2024-11-13T05:52:54,992 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:52:54,992 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,992 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/2e54af0e23894c4b989e6618322acb4b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/aa581a094fc44faeb62ad496d7562747, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/f559d2e24f8e486785b0fcd6e5a01c8f] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.6 K 2024-11-13T05:52:54,992 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting fecd5cc89f3949f9986817b49305ae0c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731477173499 2024-11-13T05:52:54,992 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e54af0e23894c4b989e6618322acb4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731477170660 2024-11-13T05:52:54,992 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting aa581a094fc44faeb62ad496d7562747, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731477171337 2024-11-13T05:52:54,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T05:52:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:54,994 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-13T05:52:54,995 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f559d2e24f8e486785b0fcd6e5a01c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731477173514 2024-11-13T05:52:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:54,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:55,023 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:55,031 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113955765d24cba45ee82571b811fcbb37a_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477173658/Put/seqid=0 2024-11-13T05:52:55,032 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/dc05375a643748a8bf829d1d0a450a7a is 50, key is test_row_0/B:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:55,039 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113a5e570bc925e4403bf8080ad12d01d35_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:55,040 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113a5e570bc925e4403bf8080ad12d01d35_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:55,040 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a5e570bc925e4403bf8080ad12d01d35_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:55,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742422_1598 (size=12304) 2024-11-13T05:52:55,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742423_1599 (size=12459) 2024-11-13T05:52:55,074 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113955765d24cba45ee82571b811fcbb37a_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113955765d24cba45ee82571b811fcbb37a_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/7bf2339bd20d4c41a0c17cac62f92562, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/7bf2339bd20d4c41a0c17cac62f92562 is 175, key is test_row_0/A:col10/1731477173658/Put/seqid=0 2024-11-13T05:52:55,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742424_1600 (size=4469) 2024-11-13T05:52:55,078 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/dc05375a643748a8bf829d1d0a450a7a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dc05375a643748a8bf829d1d0a450a7a 2024-11-13T05:52:55,079 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#504 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:55,079 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/a6bd48c907a6435283e1ebdbc4adf939 is 175, key is test_row_0/A:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:55,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742425_1601 (size=31105) 2024-11-13T05:52:55,084 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into dc05375a643748a8bf829d1d0a450a7a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:55,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:55,084 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477174989; duration=0sec 2024-11-13T05:52:55,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:55,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:52:55,084 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:55,085 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:55,085 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:52:55,086 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:55,086 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/de3fb15113be474f8a5011e60f3bcffe, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/a05f8aa1e607465f959f788744920a20, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/27d5fd6121fe46d8b352aac7b2785e20] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.6 K 2024-11-13T05:52:55,086 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting de3fb15113be474f8a5011e60f3bcffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731477170660 2024-11-13T05:52:55,086 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a05f8aa1e607465f959f788744920a20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731477171337 2024-11-13T05:52:55,087 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d5fd6121fe46d8b352aac7b2785e20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731477173514 2024-11-13T05:52:55,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742426_1602 (size=31413) 2024-11-13T05:52:55,093 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/a6bd48c907a6435283e1ebdbc4adf939 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/a6bd48c907a6435283e1ebdbc4adf939 2024-11-13T05:52:55,098 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into a6bd48c907a6435283e1ebdbc4adf939(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:55,098 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:55,098 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477174988; duration=0sec 2024-11-13T05:52:55,098 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:55,098 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:52:55,100 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:55,100 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/595e335e6a4a4e46b75e1a792ff231da is 50, key is test_row_0/C:col10/1731477173514/Put/seqid=0 2024-11-13T05:52:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742427_1603 (size=12459) 2024-11-13T05:52:55,107 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/595e335e6a4a4e46b75e1a792ff231da as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/595e335e6a4a4e46b75e1a792ff231da 2024-11-13T05:52:55,111 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into 595e335e6a4a4e46b75e1a792ff231da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:55,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:55,111 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477174989; duration=0sec 2024-11-13T05:52:55,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:55,111 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:52:55,483 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/7bf2339bd20d4c41a0c17cac62f92562 2024-11-13T05:52:55,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/ec939430f34e4d4baad3765368f58ff1 is 50, key is test_row_0/B:col10/1731477173658/Put/seqid=0 2024-11-13T05:52:55,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742428_1604 (size=12151) 2024-11-13T05:52:55,512 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/ec939430f34e4d4baad3765368f58ff1 2024-11-13T05:52:55,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/f09f5f7406354044a90a6e76e39cbe05 is 50, key is test_row_0/C:col10/1731477173658/Put/seqid=0 2024-11-13T05:52:55,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742429_1605 (size=12151) 2024-11-13T05:52:55,528 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/f09f5f7406354044a90a6e76e39cbe05 2024-11-13T05:52:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/7bf2339bd20d4c41a0c17cac62f92562 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562 2024-11-13T05:52:55,535 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562, entries=150, sequenceid=168, filesize=30.4 K 2024-11-13T05:52:55,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/ec939430f34e4d4baad3765368f58ff1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec939430f34e4d4baad3765368f58ff1 2024-11-13T05:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,538 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec939430f34e4d4baad3765368f58ff1, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T05:52:55,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/f09f5f7406354044a90a6e76e39cbe05 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/f09f5f7406354044a90a6e76e39cbe05 2024-11-13T05:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,541 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/f09f5f7406354044a90a6e76e39cbe05, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T05:52:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,541 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 636fe54399d3e25dd08446a6e1c82434 in 547ms, sequenceid=168, compaction requested=false 2024-11-13T05:52:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T05:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T05:52:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T05:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-13T05:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.9320 sec 2024-11-13T05:52:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:55,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-13T05:52:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:55,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304e2c07763134eba814b61a556c00216_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:55,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742430_1606 (size=12304) 2024-11-13T05:52:55,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731477175650, max timestamp=1731477175665Num uniques 859; 2024-11-13T05:52:55,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-13T05:52:55,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477175656, max timestamp=1731477175665Num uniques 859; 2024-11-13T05:52:55,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:55,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731477175656, max timestamp=1731477175665Num uniques 859; 2024-11-13T05:52:55,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-13T05:52:55,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477235695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477235696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477235697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T05:52:55,761 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:55,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T05:52:55,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:55,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:55,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:55,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477235812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477235812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477235812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477235821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,825 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477235831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:55,834 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:55,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:55,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:56,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304e2c07763134eba814b61a556c00216_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304e2c07763134eba814b61a556c00216_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:56,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9ef7813bb02f4dd880378e3b1157ccc4, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:56,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9ef7813bb02f4dd880378e3b1157ccc4 is 175, key is test_row_0/A:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:56,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742431_1607 (size=31105) 2024-11-13T05:52:56,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477236022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477236022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477236022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:56,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477236331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477236331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477236332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:56,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9ef7813bb02f4dd880378e3b1157ccc4 2024-11-13T05:52:56,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/58ecd192b940424daa6d6cfbf8e4f78b is 50, key is test_row_0/B:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:56,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742432_1608 (size=12151) 2024-11-13T05:52:56,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/58ecd192b940424daa6d6cfbf8e4f78b 2024-11-13T05:52:56,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/1011a363e7374f139194bd8dd3c65abd is 50, key is test_row_0/C:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:56,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742433_1609 (size=12151) 2024-11-13T05:52:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477236841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477236842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477236842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:56,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:56,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:56,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:56,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:56,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:57,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:57,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/1011a363e7374f139194bd8dd3c65abd 2024-11-13T05:52:57,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/9ef7813bb02f4dd880378e3b1157ccc4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4 2024-11-13T05:52:57,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4, entries=150, sequenceid=181, filesize=30.4 K 2024-11-13T05:52:57,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/58ecd192b940424daa6d6cfbf8e4f78b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/58ecd192b940424daa6d6cfbf8e4f78b 2024-11-13T05:52:57,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/58ecd192b940424daa6d6cfbf8e4f78b, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T05:52:57,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/1011a363e7374f139194bd8dd3c65abd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/1011a363e7374f139194bd8dd3c65abd 2024-11-13T05:52:57,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/1011a363e7374f139194bd8dd3c65abd, entries=150, sequenceid=181, filesize=11.9 K 2024-11-13T05:52:57,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 636fe54399d3e25dd08446a6e1c82434 in 1659ms, sequenceid=181, compaction requested=true 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:57,256 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:57,256 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:57,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:57,257 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:57,257 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:52:57,257 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,257 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/a6bd48c907a6435283e1ebdbc4adf939, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=91.4 K 2024-11-13T05:52:57,257 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,257 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/a6bd48c907a6435283e1ebdbc4adf939, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4] 2024-11-13T05:52:57,258 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:57,258 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:52:57,258 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,258 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dc05375a643748a8bf829d1d0a450a7a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec939430f34e4d4baad3765368f58ff1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/58ecd192b940424daa6d6cfbf8e4f78b] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.9 K 2024-11-13T05:52:57,258 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bd48c907a6435283e1ebdbc4adf939, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731477173514 2024-11-13T05:52:57,258 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting dc05375a643748a8bf829d1d0a450a7a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731477173514 2024-11-13T05:52:57,258 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf2339bd20d4c41a0c17cac62f92562, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731477173658 2024-11-13T05:52:57,259 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ec939430f34e4d4baad3765368f58ff1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731477173658 2024-11-13T05:52:57,259 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef7813bb02f4dd880378e3b1157ccc4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731477175594 2024-11-13T05:52:57,259 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ecd192b940424daa6d6cfbf8e4f78b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731477175594 2024-11-13T05:52:57,264 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:57,266 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:57,267 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/362111fe7de44a3da4b77cf0e32fa2b8 is 50, key is test_row_0/B:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:57,268 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111311a48c8353de4129af8776fc9225ff1f_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:57,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742434_1610 (size=12561) 2024-11-13T05:52:57,270 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111311a48c8353de4129af8776fc9225ff1f_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:57,270 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111311a48c8353de4129af8776fc9225ff1f_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:57,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742435_1611 (size=4469) 2024-11-13T05:52:57,273 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/362111fe7de44a3da4b77cf0e32fa2b8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/362111fe7de44a3da4b77cf0e32fa2b8 2024-11-13T05:52:57,277 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into 362111fe7de44a3da4b77cf0e32fa2b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:57,277 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:57,277 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477177256; duration=0sec 2024-11-13T05:52:57,277 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:57,277 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:52:57,277 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:57,278 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:57,278 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:52:57,278 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,278 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/595e335e6a4a4e46b75e1a792ff231da, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/f09f5f7406354044a90a6e76e39cbe05, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/1011a363e7374f139194bd8dd3c65abd] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=35.9 K 2024-11-13T05:52:57,278 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 595e335e6a4a4e46b75e1a792ff231da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731477173514 2024-11-13T05:52:57,278 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting f09f5f7406354044a90a6e76e39cbe05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731477173658 2024-11-13T05:52:57,278 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1011a363e7374f139194bd8dd3c65abd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731477175594 2024-11-13T05:52:57,283 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:57,283 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/2b26667d94794d419003332138f8c428 is 50, key is test_row_0/C:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:57,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742436_1612 (size=12561) 2024-11-13T05:52:57,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T05:52:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:57,288 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-13T05:52:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f8fcf3b3d40c4f91874e7767db2a6685_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477175690/Put/seqid=0 2024-11-13T05:52:57,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742437_1613 (size=12304) 2024-11-13T05:52:57,674 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:57,674 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/055d7172c66149ff8325647a1fced564 is 175, key is test_row_0/A:col10/1731477175594/Put/seqid=0 2024-11-13T05:52:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742438_1614 (size=31515) 2024-11-13T05:52:57,689 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/2b26667d94794d419003332138f8c428 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/2b26667d94794d419003332138f8c428 2024-11-13T05:52:57,692 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into 2b26667d94794d419003332138f8c428(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:57,692 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:57,692 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477177256; duration=0sec 2024-11-13T05:52:57,692 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:57,693 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:52:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:57,699 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f8fcf3b3d40c4f91874e7767db2a6685_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8fcf3b3d40c4f91874e7767db2a6685_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:57,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8b58d236b04b47f4b833cf6699f6d578, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8b58d236b04b47f4b833cf6699f6d578 is 175, key is test_row_0/A:col10/1731477175690/Put/seqid=0 2024-11-13T05:52:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742439_1615 (size=31105) 2024-11-13T05:52:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:57,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:52:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477237854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477237855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477237856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477237858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477237862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,864 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477237972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477237972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477237972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477237972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,081 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/055d7172c66149ff8325647a1fced564 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/055d7172c66149ff8325647a1fced564 2024-11-13T05:52:58,084 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into 055d7172c66149ff8325647a1fced564(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:52:58,084 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:58,084 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477177256; duration=0sec 2024-11-13T05:52:58,084 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:58,084 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:52:58,103 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8b58d236b04b47f4b833cf6699f6d578 2024-11-13T05:52:58,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/e78cf2ed61794a11845b51552d75d636 is 50, key is test_row_0/B:col10/1731477175690/Put/seqid=0 2024-11-13T05:52:58,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742440_1616 (size=12151) 2024-11-13T05:52:58,112 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/e78cf2ed61794a11845b51552d75d636 2024-11-13T05:52:58,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/9ee401da56d649a4bdc495069a3d1fcd is 50, key is test_row_0/C:col10/1731477175690/Put/seqid=0 2024-11-13T05:52:58,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742441_1617 (size=12151) 2024-11-13T05:52:58,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477238182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477238182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477238182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477238182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477238491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477238491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477238492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477238492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:58,520 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/9ee401da56d649a4bdc495069a3d1fcd 2024-11-13T05:52:58,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/8b58d236b04b47f4b833cf6699f6d578 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578 2024-11-13T05:52:58,526 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578, entries=150, sequenceid=219, filesize=30.4 K 2024-11-13T05:52:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/e78cf2ed61794a11845b51552d75d636 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e78cf2ed61794a11845b51552d75d636 2024-11-13T05:52:58,529 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e78cf2ed61794a11845b51552d75d636, entries=150, sequenceid=219, filesize=11.9 K 2024-11-13T05:52:58,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/9ee401da56d649a4bdc495069a3d1fcd as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9ee401da56d649a4bdc495069a3d1fcd 2024-11-13T05:52:58,532 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9ee401da56d649a4bdc495069a3d1fcd, entries=150, sequenceid=219, filesize=11.9 K 2024-11-13T05:52:58,533 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 636fe54399d3e25dd08446a6e1c82434 in 1245ms, sequenceid=219, compaction requested=false 2024-11-13T05:52:58,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:58,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:58,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T05:52:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T05:52:58,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T05:52:58,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7700 sec 2024-11-13T05:52:58,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7740 sec 2024-11-13T05:52:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:59,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-13T05:52:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:59,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:52:59,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111300e474c52ca44b59a58202c45e19fc52_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477179002/Put/seqid=0 2024-11-13T05:52:59,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731477179018, max timestamp=1731477179025Num uniques 859; 2024-11-13T05:52:59,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T05:52:59,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731477179018, max timestamp=1731477179025Num uniques 859; 2024-11-13T05:52:59,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731477179018, max timestamp=1731477179025Num uniques 859; 2024-11-13T05:52:59,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T05:52:59,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/bc386b7bf2fe:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-13T05:52:59,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742442_1618 (size=12304) 2024-11-13T05:52:59,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477239046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477239047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477239048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477239048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477239161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477239161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477239162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477239162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477239371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477239371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477239372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477239372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:59,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111300e474c52ca44b59a58202c45e19fc52_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111300e474c52ca44b59a58202c45e19fc52_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:59,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/dbe59616ec624c3cb04626b163f32e51, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:59,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/dbe59616ec624c3cb04626b163f32e51 is 175, key is test_row_0/A:col10/1731477179002/Put/seqid=0 2024-11-13T05:52:59,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742443_1619 (size=31105) 2024-11-13T05:52:59,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/dbe59616ec624c3cb04626b163f32e51 2024-11-13T05:52:59,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/8b28bd1d233542828ce7ec6b0d1de54c is 50, key is test_row_0/B:col10/1731477179002/Put/seqid=0 2024-11-13T05:52:59,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742444_1620 (size=12151) 2024-11-13T05:52:59,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/8b28bd1d233542828ce7ec6b0d1de54c 2024-11-13T05:52:59,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/7bd78d23021245a3a8e1b32ede7ebd42 is 50, key is test_row_0/C:col10/1731477179002/Put/seqid=0 2024-11-13T05:52:59,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742445_1621 (size=12151) 2024-11-13T05:52:59,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/7bd78d23021245a3a8e1b32ede7ebd42 2024-11-13T05:52:59,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/dbe59616ec624c3cb04626b163f32e51 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51 2024-11-13T05:52:59,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51, entries=150, sequenceid=233, filesize=30.4 K 2024-11-13T05:52:59,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/8b28bd1d233542828ce7ec6b0d1de54c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/8b28bd1d233542828ce7ec6b0d1de54c 2024-11-13T05:52:59,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/8b28bd1d233542828ce7ec6b0d1de54c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T05:52:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/7bd78d23021245a3a8e1b32ede7ebd42 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/7bd78d23021245a3a8e1b32ede7ebd42 2024-11-13T05:52:59,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/7bd78d23021245a3a8e1b32ede7ebd42, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T05:52:59,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 636fe54399d3e25dd08446a6e1c82434 in 705ms, sequenceid=233, compaction requested=true 2024-11-13T05:52:59,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:52:59,709 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:59,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:52:59,710 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:59,710 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:52:59,710 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:59,710 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/055d7172c66149ff8325647a1fced564, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=91.5 K 2024-11-13T05:52:59,710 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:59,710 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/055d7172c66149ff8325647a1fced564, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51] 2024-11-13T05:52:59,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:59,710 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:52:59,710 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055d7172c66149ff8325647a1fced564, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731477175594 2024-11-13T05:52:59,711 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b58d236b04b47f4b833cf6699f6d578, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731477175690 2024-11-13T05:52:59,711 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:52:59,711 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:52:59,711 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:52:59,711 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/362111fe7de44a3da4b77cf0e32fa2b8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e78cf2ed61794a11845b51552d75d636, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/8b28bd1d233542828ce7ec6b0d1de54c] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=36.0 K 2024-11-13T05:52:59,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:52:59,711 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 362111fe7de44a3da4b77cf0e32fa2b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731477175594 2024-11-13T05:52:59,712 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe59616ec624c3cb04626b163f32e51, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477177854 2024-11-13T05:52:59,712 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting e78cf2ed61794a11845b51552d75d636, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731477175690 2024-11-13T05:52:59,712 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b28bd1d233542828ce7ec6b0d1de54c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477177854 2024-11-13T05:52:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:52:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:52:59,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:52:59,731 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:59,732 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/b5aaf6760a024ab48a1aabeaf4bc2859 is 50, key is test_row_0/B:col10/1731477179002/Put/seqid=0 2024-11-13T05:52:59,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-13T05:52:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:52:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:52:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:52:59,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-13T05:52:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:59,758 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:59,787 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cefc01d6001b45a78c7fd0a17755fa28_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:59,790 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cefc01d6001b45a78c7fd0a17755fa28_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:59,790 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cefc01d6001b45a78c7fd0a17755fa28_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:59,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f2b403317a84cb7ba8aecadfe6e67cf_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477179728/Put/seqid=0 2024-11-13T05:52:59,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742446_1622 (size=12663) 2024-11-13T05:52:59,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477239794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477239794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477239796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477239796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742447_1623 (size=4469) 2024-11-13T05:52:59,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742448_1624 (size=14994) 2024-11-13T05:52:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477239881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:52:59,896 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:59,896 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:52:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:52:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:52:59,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f2b403317a84cb7ba8aecadfe6e67cf_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f2b403317a84cb7ba8aecadfe6e67cf_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:52:59,899 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#523 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:52:59,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/1d66926fede6494aba799d8277eee431, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:52:59,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/1d66926fede6494aba799d8277eee431 is 175, key is test_row_0/A:col10/1731477179728/Put/seqid=0 2024-11-13T05:52:59,900 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/6720a2d9acb64573b8bf7b7e4a604903 is 175, key is test_row_0/A:col10/1731477179002/Put/seqid=0 2024-11-13T05:52:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T05:52:59,922 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:52:59,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T05:52:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-13T05:52:59,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T05:52:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:52:59,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T05:52:59,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T05:52:59,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742449_1625 (size=31617) 2024-11-13T05:53:00,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477239952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477239972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742450_1626 (size=39949) 2024-11-13T05:53:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477240006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477240006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:00,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,199 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/b5aaf6760a024ab48a1aabeaf4bc2859 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/b5aaf6760a024ab48a1aabeaf4bc2859 2024-11-13T05:53:00,218 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into b5aaf6760a024ab48a1aabeaf4bc2859(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:00,218 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:00,218 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477179710; duration=0sec 2024-11-13T05:53:00,218 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:53:00,218 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:53:00,218 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:00,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:00,219 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:53:00,219 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,219 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/2b26667d94794d419003332138f8c428, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9ee401da56d649a4bdc495069a3d1fcd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/7bd78d23021245a3a8e1b32ede7ebd42] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=36.0 K 2024-11-13T05:53:00,220 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b26667d94794d419003332138f8c428, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731477175594 2024-11-13T05:53:00,220 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ee401da56d649a4bdc495069a3d1fcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731477175690 2024-11-13T05:53:00,220 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd78d23021245a3a8e1b32ede7ebd42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477177854 2024-11-13T05:53:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477240212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477240213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477240217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477240213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,232 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:00,232 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/74ae82e181c54e8b95f0214c8983b7c5 is 50, key is test_row_0/C:col10/1731477179002/Put/seqid=0 2024-11-13T05:53:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:00,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742451_1627 (size=12663) 2024-11-13T05:53:00,247 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/74ae82e181c54e8b95f0214c8983b7c5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/74ae82e181c54e8b95f0214c8983b7c5 2024-11-13T05:53:00,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,255 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into 74ae82e181c54e8b95f0214c8983b7c5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:00,255 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:00,255 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477179712; duration=0sec 2024-11-13T05:53:00,255 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:00,255 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:53:00,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/1d66926fede6494aba799d8277eee431 2024-11-13T05:53:00,415 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/6720a2d9acb64573b8bf7b7e4a604903 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6720a2d9acb64573b8bf7b7e4a604903 2024-11-13T05:53:00,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/a4f21c5f2b0942fe94839f00e3037b33 is 50, key is test_row_0/B:col10/1731477179728/Put/seqid=0 2024-11-13T05:53:00,419 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into 6720a2d9acb64573b8bf7b7e4a604903(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:00,419 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:00,420 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477179708; duration=0sec 2024-11-13T05:53:00,420 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:00,420 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:53:00,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742452_1628 (size=12301) 2024-11-13T05:53:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:00,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:00,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477240532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477240532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477240532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477240532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:00,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/a4f21c5f2b0942fe94839f00e3037b33 2024-11-13T05:53:00,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:00,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:00,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:00,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/447ff5ced68c458e949ebacdbaaa4997 is 50, key is test_row_0/C:col10/1731477179728/Put/seqid=0 2024-11-13T05:53:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742453_1629 (size=12301) 2024-11-13T05:53:00,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/447ff5ced68c458e949ebacdbaaa4997 2024-11-13T05:53:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/1d66926fede6494aba799d8277eee431 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431 2024-11-13T05:53:00,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431, entries=200, sequenceid=267, filesize=39.0 K 2024-11-13T05:53:00,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/a4f21c5f2b0942fe94839f00e3037b33 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a4f21c5f2b0942fe94839f00e3037b33 2024-11-13T05:53:01,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a4f21c5f2b0942fe94839f00e3037b33, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T05:53:01,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/447ff5ced68c458e949ebacdbaaa4997 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/447ff5ced68c458e949ebacdbaaa4997 2024-11-13T05:53:01,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/447ff5ced68c458e949ebacdbaaa4997, entries=150, sequenceid=267, filesize=12.0 K 2024-11-13T05:53:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:01,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=60.38 KB/61830 for 636fe54399d3e25dd08446a6e1c82434 in 1364ms, sequenceid=267, compaction requested=false 2024-11-13T05:53:01,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:01,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:53:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:53:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:53:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:53:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:01,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113524351604b86498cb69fbf72c84bf8db_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:01,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742454_1630 (size=12454) 2024-11-13T05:53:01,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:01,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:01,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477241326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477241372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477241398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477241398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477241502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:53:01,588 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113524351604b86498cb69fbf72c84bf8db_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113524351604b86498cb69fbf72c84bf8db_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:01,589 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/d8fbac8e73514e0ca48307faa9adad3a, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:01,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/d8fbac8e73514e0ca48307faa9adad3a is 175, key is test_row_0/A:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:01,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742455_1631 (size=31255) 2024-11-13T05:53:01,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477241751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477241751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477241751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477241751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:01,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:01,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:01,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,020 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/d8fbac8e73514e0ca48307faa9adad3a 2024-11-13T05:53:02,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/84be606fb09b48f0b3c3a4475b47267e is 50, key is test_row_0/B:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:02,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742456_1632 (size=12301) 2024-11-13T05:53:02,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477242061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477242061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477242062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477242062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:02,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/84be606fb09b48f0b3c3a4475b47267e 2024-11-13T05:53:02,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/bc274652c0ac4b0aa48c127ce6bd2869 is 50, key is test_row_0/C:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:02,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742457_1633 (size=12301) 2024-11-13T05:53:02,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477242572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477242581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477242581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477242581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:02,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:02,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:53:02,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/bc274652c0ac4b0aa48c127ce6bd2869 2024-11-13T05:53:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/d8fbac8e73514e0ca48307faa9adad3a as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a 2024-11-13T05:53:02,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a, entries=150, sequenceid=284, filesize=30.5 K 2024-11-13T05:53:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/84be606fb09b48f0b3c3a4475b47267e as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/84be606fb09b48f0b3c3a4475b47267e 2024-11-13T05:53:02,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/84be606fb09b48f0b3c3a4475b47267e, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T05:53:02,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/bc274652c0ac4b0aa48c127ce6bd2869 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/bc274652c0ac4b0aa48c127ce6bd2869 2024-11-13T05:53:02,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/bc274652c0ac4b0aa48c127ce6bd2869, entries=150, sequenceid=284, filesize=12.0 K 2024-11-13T05:53:02,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 636fe54399d3e25dd08446a6e1c82434 in 1774ms, sequenceid=284, compaction requested=true 2024-11-13T05:53:02,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:53:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:53:02,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:53:02,883 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,883 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,883 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6720a2d9acb64573b8bf7b7e4a604903, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=100.4 K 2024-11-13T05:53:02,883 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/b5aaf6760a024ab48a1aabeaf4bc2859, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a4f21c5f2b0942fe94839f00e3037b33, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/84be606fb09b48f0b3c3a4475b47267e] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=36.4 K 2024-11-13T05:53:02,883 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,883 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6720a2d9acb64573b8bf7b7e4a604903, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a] 2024-11-13T05:53:02,884 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5aaf6760a024ab48a1aabeaf4bc2859, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477177854 2024-11-13T05:53:02,884 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 6720a2d9acb64573b8bf7b7e4a604903, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477177854 2024-11-13T05:53:02,884 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4f21c5f2b0942fe94839f00e3037b33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731477179039 2024-11-13T05:53:02,884 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d66926fede6494aba799d8277eee431, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731477179039 2024-11-13T05:53:02,884 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84be606fb09b48f0b3c3a4475b47267e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731477179793 2024-11-13T05:53:02,884 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting d8fbac8e73514e0ca48307faa9adad3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731477179793 2024-11-13T05:53:02,891 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:02,891 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/dff1233adc484ace8913a6ce258b24f5 is 50, key is test_row_0/B:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:02,892 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:02,894 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c7b6d8537e474b6d9c8497a384c3c72d_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:02,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742458_1634 (size=12915) 2024-11-13T05:53:02,895 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c7b6d8537e474b6d9c8497a384c3c72d_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:02,896 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c7b6d8537e474b6d9c8497a384c3c72d_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:02,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742459_1635 (size=4469) 2024-11-13T05:53:02,899 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#532 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:02,899 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/6ddc7e9ecc8a46799ba92f7ea676ba01 is 175, key is test_row_0/A:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:02,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742460_1636 (size=31869) 2024-11-13T05:53:02,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38979 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:02,924 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:53:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:02,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132f64153a48bd467cab475a3df291733b_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477181331/Put/seqid=0 2024-11-13T05:53:02,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742461_1637 (size=12454) 2024-11-13T05:53:03,298 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/dff1233adc484ace8913a6ce258b24f5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dff1233adc484ace8913a6ce258b24f5 2024-11-13T05:53:03,301 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into dff1233adc484ace8913a6ce258b24f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:03,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:03,301 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477182883; duration=0sec 2024-11-13T05:53:03,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:53:03,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:53:03,301 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:03,302 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:03,302 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:53:03,302 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:03,302 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/74ae82e181c54e8b95f0214c8983b7c5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/447ff5ced68c458e949ebacdbaaa4997, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/bc274652c0ac4b0aa48c127ce6bd2869] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=36.4 K 2024-11-13T05:53:03,302 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ae82e181c54e8b95f0214c8983b7c5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731477177854 2024-11-13T05:53:03,303 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447ff5ced68c458e949ebacdbaaa4997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731477179039 2024-11-13T05:53:03,303 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc274652c0ac4b0aa48c127ce6bd2869, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731477179793 2024-11-13T05:53:03,309 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#534 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:03,309 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/6ddc7e9ecc8a46799ba92f7ea676ba01 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6ddc7e9ecc8a46799ba92f7ea676ba01 2024-11-13T05:53:03,309 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/db93a2e3ecad41269b1fa7607c1d81ef is 50, key is test_row_0/C:col10/1731477181108/Put/seqid=0 2024-11-13T05:53:03,312 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into 6ddc7e9ecc8a46799ba92f7ea676ba01(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:03,312 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:03,312 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477182882; duration=0sec 2024-11-13T05:53:03,312 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:03,313 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:53:03,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742462_1638 (size=12915) 2024-11-13T05:53:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:53:03,335 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132f64153a48bd467cab475a3df291733b_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f64153a48bd467cab475a3df291733b_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/399ff72e3d0c45c1bcff19f38b01fd84, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:03,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/399ff72e3d0c45c1bcff19f38b01fd84 is 175, key is test_row_0/A:col10/1731477181331/Put/seqid=0 2024-11-13T05:53:03,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742463_1639 (size=31255) 2024-11-13T05:53:03,339 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/399ff72e3d0c45c1bcff19f38b01fd84 2024-11-13T05:53:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/a44e30dce7ef487594ec5c2505e32817 is 50, key is test_row_0/B:col10/1731477181331/Put/seqid=0 2024-11-13T05:53:03,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742464_1640 (size=12301) 2024-11-13T05:53:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:03,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. as already flushing 2024-11-13T05:53:03,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477243603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477243604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477243605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477243606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,717 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/db93a2e3ecad41269b1fa7607c1d81ef as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/db93a2e3ecad41269b1fa7607c1d81ef 2024-11-13T05:53:03,721 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into db93a2e3ecad41269b1fa7607c1d81ef(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:03,721 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:03,721 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477182883; duration=0sec 2024-11-13T05:53:03,721 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:03,721 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:53:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477243722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,761 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/a44e30dce7ef487594ec5c2505e32817 2024-11-13T05:53:03,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/126dfd195fab43f7878c2db6eb4ab7e6 is 50, key is test_row_0/C:col10/1731477181331/Put/seqid=0 2024-11-13T05:53:03,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742465_1641 (size=12301) 2024-11-13T05:53:03,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477243931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477243932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477243932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477243932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:03,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1731477243941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:03,943 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38979,1731477016232 for 'test_row_0' in 636fe54399d3e25dd08446a6e1c82434 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-13T05:53:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:04,169 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/126dfd195fab43f7878c2db6eb4ab7e6 2024-11-13T05:53:04,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/399ff72e3d0c45c1bcff19f38b01fd84 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84 2024-11-13T05:53:04,174 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84, entries=150, sequenceid=304, filesize=30.5 K 2024-11-13T05:53:04,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/a44e30dce7ef487594ec5c2505e32817 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a44e30dce7ef487594ec5c2505e32817 2024-11-13T05:53:04,178 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a44e30dce7ef487594ec5c2505e32817, entries=150, sequenceid=304, filesize=12.0 K 2024-11-13T05:53:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/126dfd195fab43f7878c2db6eb4ab7e6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/126dfd195fab43f7878c2db6eb4ab7e6 2024-11-13T05:53:04,182 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/126dfd195fab43f7878c2db6eb4ab7e6, entries=150, sequenceid=304, filesize=12.0 K 2024-11-13T05:53:04,182 INFO [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 636fe54399d3e25dd08446a6e1c82434 in 1258ms, sequenceid=304, compaction requested=false 2024-11-13T05:53:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-13T05:53:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-13T05:53:04,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T05:53:04,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2580 sec 2024-11-13T05:53:04,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 4.2680 sec 2024-11-13T05:53:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:04,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-13T05:53:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:53:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:53:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:53:04,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:04,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137c9ae97da7c74bbe8dbea82d6430a074_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:04,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742466_1642 (size=12454) 2024-11-13T05:53:04,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477244263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477244264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477244265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477244268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477244371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477244381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477244381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477244381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477244581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477244591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477244591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477244591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:53:04,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411137c9ae97da7c74bbe8dbea82d6430a074_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137c9ae97da7c74bbe8dbea82d6430a074_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:04,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/93c9feac17ad4158b399aec8d62d9354, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:04,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/93c9feac17ad4158b399aec8d62d9354 is 175, key is test_row_0/A:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:04,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742467_1643 (size=31255) 2024-11-13T05:53:04,664 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/93c9feac17ad4158b399aec8d62d9354 2024-11-13T05:53:04,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/0dc81dfdf7e24de2ae580bdd99d8b383 is 50, key is test_row_0/B:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:04,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742468_1644 (size=12301) 2024-11-13T05:53:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477244892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477244901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477244901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477244901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:04,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:05,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/0dc81dfdf7e24de2ae580bdd99d8b383 2024-11-13T05:53:05,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/ee93c66bb5314d7a868e940a895f0842 is 50, key is test_row_0/C:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:05,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742469_1645 (size=12301) 2024-11-13T05:53:05,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58456 deadline: 1731477245401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:05,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:05,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1731477245411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:05,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1731477245411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:05,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:53:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38979 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1731477245411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 , the old value is region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:53:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., hostname=bc386b7bf2fe,38979,1731477016232, seqNum=5 because the exception is null or not the one we care about 2024-11-13T05:53:05,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/ee93c66bb5314d7a868e940a895f0842 2024-11-13T05:53:05,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/93c9feac17ad4158b399aec8d62d9354 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354 2024-11-13T05:53:05,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354, entries=150, sequenceid=321, filesize=30.5 K 2024-11-13T05:53:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/0dc81dfdf7e24de2ae580bdd99d8b383 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0dc81dfdf7e24de2ae580bdd99d8b383 2024-11-13T05:53:05,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0dc81dfdf7e24de2ae580bdd99d8b383, entries=150, sequenceid=321, filesize=12.0 K 2024-11-13T05:53:05,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/ee93c66bb5314d7a868e940a895f0842 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/ee93c66bb5314d7a868e940a895f0842 2024-11-13T05:53:05,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/ee93c66bb5314d7a868e940a895f0842, entries=150, sequenceid=321, filesize=12.0 K 2024-11-13T05:53:05,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 636fe54399d3e25dd08446a6e1c82434 in 1257ms, sequenceid=321, compaction requested=true 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:05,501 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 636fe54399d3e25dd08446a6e1c82434:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T05:53:05,501 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:05,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:53:05,502 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:05,502 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/A is initiating minor compaction (all files) 2024-11-13T05:53:05,502 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/A in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:05,502 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6ddc7e9ecc8a46799ba92f7ea676ba01, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=92.2 K 2024-11-13T05:53:05,502 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:05,502 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. files: [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6ddc7e9ecc8a46799ba92f7ea676ba01, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354] 2024-11-13T05:53:05,502 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:05,502 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/B is initiating minor compaction (all files) 2024-11-13T05:53:05,502 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/B in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:05,502 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ddc7e9ecc8a46799ba92f7ea676ba01, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731477179793 2024-11-13T05:53:05,502 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dff1233adc484ace8913a6ce258b24f5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a44e30dce7ef487594ec5c2505e32817, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0dc81dfdf7e24de2ae580bdd99d8b383] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=36.6 K 2024-11-13T05:53:05,503 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 399ff72e3d0c45c1bcff19f38b01fd84, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731477181331 2024-11-13T05:53:05,503 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting dff1233adc484ace8913a6ce258b24f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731477179793 2024-11-13T05:53:05,503 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c9feac17ad4158b399aec8d62d9354, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731477183604 2024-11-13T05:53:05,503 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting a44e30dce7ef487594ec5c2505e32817, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731477181331 2024-11-13T05:53:05,503 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc81dfdf7e24de2ae580bdd99d8b383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731477183604 2024-11-13T05:53:05,506 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:05,507 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#B#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:05,508 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/579b1f60db124826a48617653ff8faa6 is 50, key is test_row_0/B:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:05,509 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411134f853a7a7bab4ba19951847698b71ae9_636fe54399d3e25dd08446a6e1c82434 store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:05,511 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411134f853a7a7bab4ba19951847698b71ae9_636fe54399d3e25dd08446a6e1c82434, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:05,511 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134f853a7a7bab4ba19951847698b71ae9_636fe54399d3e25dd08446a6e1c82434 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:05,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742470_1646 (size=13017) 2024-11-13T05:53:05,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742471_1647 (size=4469) 2024-11-13T05:53:05,915 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/579b1f60db124826a48617653ff8faa6 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/579b1f60db124826a48617653ff8faa6 2024-11-13T05:53:05,917 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#A#compaction#541 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:05,917 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/7f3b1afe86fb40739f829916dfd9c023 is 175, key is test_row_0/A:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:05,919 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/B of 636fe54399d3e25dd08446a6e1c82434 into 579b1f60db124826a48617653ff8faa6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:05,919 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:05,919 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/B, priority=13, startTime=1731477185501; duration=0sec 2024-11-13T05:53:05,919 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T05:53:05,919 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:B 2024-11-13T05:53:05,919 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T05:53:05,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742472_1648 (size=31971) 2024-11-13T05:53:05,920 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T05:53:05,920 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1541): 636fe54399d3e25dd08446a6e1c82434/C is initiating minor compaction (all files) 2024-11-13T05:53:05,920 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 636fe54399d3e25dd08446a6e1c82434/C in TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:05,920 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/db93a2e3ecad41269b1fa7607c1d81ef, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/126dfd195fab43f7878c2db6eb4ab7e6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/ee93c66bb5314d7a868e940a895f0842] into tmpdir=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp, totalSize=36.6 K 2024-11-13T05:53:05,921 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting db93a2e3ecad41269b1fa7607c1d81ef, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731477179793 2024-11-13T05:53:05,921 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting 126dfd195fab43f7878c2db6eb4ab7e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731477181331 2024-11-13T05:53:05,921 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] compactions.Compactor(225): Compacting ee93c66bb5314d7a868e940a895f0842, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731477183604 2024-11-13T05:53:05,931 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 636fe54399d3e25dd08446a6e1c82434#C#compaction#542 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:53:05,931 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/0a31888562214e699de54e24d3f3301b is 50, key is test_row_0/C:col10/1731477184244/Put/seqid=0 2024-11-13T05:53:05,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742473_1649 (size=13017) 2024-11-13T05:53:06,149 INFO [Thread-2504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2504. 2024-11-13T05:53:06,149 DEBUG [Thread-2504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,149 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,149 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,149 INFO [Thread-2502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2502. 2024-11-13T05:53:06,149 DEBUG [Thread-2502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,149 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,150 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,151 INFO [Thread-2500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2500. 2024-11-13T05:53:06,151 DEBUG [Thread-2500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,151 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,151 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,151 INFO [Thread-2506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2506. 2024-11-13T05:53:06,151 DEBUG [Thread-2506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,151 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,151 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,152 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-11-13T05:53:06,152 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,152 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,152 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,331 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/7f3b1afe86fb40739f829916dfd9c023 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7f3b1afe86fb40739f829916dfd9c023 2024-11-13T05:53:06,336 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/A of 636fe54399d3e25dd08446a6e1c82434 into 7f3b1afe86fb40739f829916dfd9c023(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:06,336 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:06,336 INFO [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/A, priority=13, startTime=1731477185501; duration=0sec 2024-11-13T05:53:06,336 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:06,336 DEBUG [RS:0;bc386b7bf2fe:38979-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:A 2024-11-13T05:53:06,339 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/0a31888562214e699de54e24d3f3301b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/0a31888562214e699de54e24d3f3301b 2024-11-13T05:53:06,343 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 636fe54399d3e25dd08446a6e1c82434/C of 636fe54399d3e25dd08446a6e1c82434 into 0a31888562214e699de54e24d3f3301b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:53:06,344 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:06,344 INFO [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., storeName=636fe54399d3e25dd08446a6e1c82434/C, priority=13, startTime=1731477185501; duration=0sec 2024-11-13T05:53:06,344 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T05:53:06,344 DEBUG [RS:0;bc386b7bf2fe:38979-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 636fe54399d3e25dd08446a6e1c82434:C 2024-11-13T05:53:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38979 {}] regionserver.HRegion(8855): Flush requested on 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:06,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-13T05:53:06,424 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-11-13T05:53:06,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:53:06,424 INFO [Thread-2493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2493. 2024-11-13T05:53:06,425 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:06,425 DEBUG [Thread-2493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,425 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:53:06,425 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:53:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:06,425 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,425 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d6191af4f3fd4e0d82193b92b6b4ca87_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477184262/Put/seqid=0 2024-11-13T05:53:06,434 INFO [Thread-2489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2489. 2024-11-13T05:53:06,434 INFO [Thread-2487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2487. 2024-11-13T05:53:06,434 DEBUG [Thread-2489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,434 DEBUG [Thread-2487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:06,434 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,434 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,435 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,435 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:06,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:06,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742474_1650 (size=12454) 2024-11-13T05:53:06,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:53:06,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d6191af4f3fd4e0d82193b92b6b4ca87_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d6191af4f3fd4e0d82193b92b6b4ca87_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:06,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/775f052707104ce39a230156de2b24f1, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:06,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/775f052707104ce39a230156de2b24f1 is 175, key is test_row_0/A:col10/1731477184262/Put/seqid=0 2024-11-13T05:53:06,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742475_1651 (size=31255) 2024-11-13T05:53:07,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/775f052707104ce39a230156de2b24f1 2024-11-13T05:53:07,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/0afc710896a14338ade0d4a18b6dd040 is 50, key is test_row_0/B:col10/1731477184262/Put/seqid=0 2024-11-13T05:53:07,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742476_1652 (size=12301) 2024-11-13T05:53:07,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/0afc710896a14338ade0d4a18b6dd040 2024-11-13T05:53:07,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/469982ea945645baaeaff3bb6d9ba70d is 50, key is test_row_0/C:col10/1731477184262/Put/seqid=0 2024-11-13T05:53:07,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742477_1653 (size=12301) 2024-11-13T05:53:08,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/469982ea945645baaeaff3bb6d9ba70d 2024-11-13T05:53:08,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/775f052707104ce39a230156de2b24f1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/775f052707104ce39a230156de2b24f1 2024-11-13T05:53:08,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/775f052707104ce39a230156de2b24f1, entries=150, sequenceid=344, filesize=30.5 K 2024-11-13T05:53:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/0afc710896a14338ade0d4a18b6dd040 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0afc710896a14338ade0d4a18b6dd040 2024-11-13T05:53:08,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0afc710896a14338ade0d4a18b6dd040, entries=150, sequenceid=344, filesize=12.0 K 2024-11-13T05:53:08,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/469982ea945645baaeaff3bb6d9ba70d as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/469982ea945645baaeaff3bb6d9ba70d 2024-11-13T05:53:08,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/469982ea945645baaeaff3bb6d9ba70d, entries=150, sequenceid=344, filesize=12.0 K 2024-11-13T05:53:08,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 636fe54399d3e25dd08446a6e1c82434 in 1672ms, sequenceid=344, compaction requested=false 2024-11-13T05:53:08,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 636fe54399d3e25dd08446a6e1c82434: 2024-11-13T05:53:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T05:53:08,112 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:53:13,984 INFO [Thread-2495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2495. 2024-11-13T05:53:13,984 DEBUG [Thread-2495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T05:53:13,984 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:13,984 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:13,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2681 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2686 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2646 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2678 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2686 2024-11-13T05:53:13,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T05:53:13,984 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:53:13,984 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:53:13,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:13,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:13,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:13,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T05:53:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T05:53:13,989 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477193989"}]},"ts":"1731477193989"} 2024-11-13T05:53:13,991 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T05:53:13,991 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T05:53:13,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T05:53:13,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, UNASSIGN}] 2024-11-13T05:53:13,993 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, UNASSIGN 2024-11-13T05:53:13,994 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=CLOSING, regionLocation=bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:13,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, UNASSIGN because future has completed 2024-11-13T05:53:13,997 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:53:13,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232}] 2024-11-13T05:53:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T05:53:14,149 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:14,149 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 636fe54399d3e25dd08446a6e1c82434, disabling compactions & flushes 2024-11-13T05:53:14,150 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. after waiting 0 ms 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:14,150 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 636fe54399d3e25dd08446a6e1c82434 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=A 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=B 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 636fe54399d3e25dd08446a6e1c82434, store=C 2024-11-13T05:53:14,150 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-13T05:53:14,156 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113826c1084c7fd40528c3561c949640139_636fe54399d3e25dd08446a6e1c82434 is 50, key is test_row_0/A:col10/1731477193982/Put/seqid=0 2024-11-13T05:53:14,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742478_1654 (size=9914) 2024-11-13T05:53:14,160 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:53:14,167 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113826c1084c7fd40528c3561c949640139_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113826c1084c7fd40528c3561c949640139_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:14,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4424c6e8320f4333ace5dba2f1233df1, store: [table=TestAcidGuarantees family=A region=636fe54399d3e25dd08446a6e1c82434] 2024-11-13T05:53:14,168 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4424c6e8320f4333ace5dba2f1233df1 is 175, key is test_row_0/A:col10/1731477193982/Put/seqid=0 2024-11-13T05:53:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742479_1655 (size=22561) 2024-11-13T05:53:14,172 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4424c6e8320f4333ace5dba2f1233df1 2024-11-13T05:53:14,179 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/554544fd5da74990a30982c001a62e30 is 50, key is test_row_0/B:col10/1731477193982/Put/seqid=0 2024-11-13T05:53:14,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742480_1656 (size=9857) 2024-11-13T05:53:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T05:53:14,496 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:53:14,583 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/554544fd5da74990a30982c001a62e30 2024-11-13T05:53:14,591 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/e797384c3ab6471a90bfebd5822f26b4 is 50, key is test_row_0/C:col10/1731477193982/Put/seqid=0 2024-11-13T05:53:14,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742481_1657 (size=9857) 2024-11-13T05:53:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T05:53:14,999 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/e797384c3ab6471a90bfebd5822f26b4 2024-11-13T05:53:15,002 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/A/4424c6e8320f4333ace5dba2f1233df1 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4424c6e8320f4333ace5dba2f1233df1 2024-11-13T05:53:15,006 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4424c6e8320f4333ace5dba2f1233df1, entries=100, sequenceid=350, filesize=22.0 K 2024-11-13T05:53:15,006 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/B/554544fd5da74990a30982c001a62e30 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/554544fd5da74990a30982c001a62e30 2024-11-13T05:53:15,009 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/554544fd5da74990a30982c001a62e30, entries=100, sequenceid=350, filesize=9.6 K 2024-11-13T05:53:15,009 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/.tmp/C/e797384c3ab6471a90bfebd5822f26b4 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/e797384c3ab6471a90bfebd5822f26b4 2024-11-13T05:53:15,012 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/e797384c3ab6471a90bfebd5822f26b4, entries=100, sequenceid=350, filesize=9.6 K 2024-11-13T05:53:15,013 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 636fe54399d3e25dd08446a6e1c82434 in 863ms, sequenceid=350, compaction requested=true 2024-11-13T05:53:15,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/87cb7745d1c6484a9c08f82df744830c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/44335731470f4599b1475e1476bf6024, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/a6bd48c907a6435283e1ebdbc4adf939, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/055d7172c66149ff8325647a1fced564, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6720a2d9acb64573b8bf7b7e4a604903, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6ddc7e9ecc8a46799ba92f7ea676ba01, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354] to archive 2024-11-13T05:53:15,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:53:15,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4735a1d6aa064be482f681c9326db7b4 2024-11-13T05:53:15,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/f05a065784404a60a0c28a9869a7ac1c 2024-11-13T05:53:15,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/87cb7745d1c6484a9c08f82df744830c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/87cb7745d1c6484a9c08f82df744830c 2024-11-13T05:53:15,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9f7d2c67ab374ea4bfc63dfa402bbdf3 2024-11-13T05:53:15,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5d8d74e4b1fe4dceb1abd9b4d485b862 2024-11-13T05:53:15,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8398e27574284ea1912649375018a4b8 2024-11-13T05:53:15,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/44335731470f4599b1475e1476bf6024 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/44335731470f4599b1475e1476bf6024 2024-11-13T05:53:15,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/5ac815f15e94495db955ac3be922e603 2024-11-13T05:53:15,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/fecd5cc89f3949f9986817b49305ae0c 2024-11-13T05:53:15,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/a6bd48c907a6435283e1ebdbc4adf939 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/a6bd48c907a6435283e1ebdbc4adf939 2024-11-13T05:53:15,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7bf2339bd20d4c41a0c17cac62f92562 2024-11-13T05:53:15,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/055d7172c66149ff8325647a1fced564 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/055d7172c66149ff8325647a1fced564 2024-11-13T05:53:15,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/9ef7813bb02f4dd880378e3b1157ccc4 2024-11-13T05:53:15,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/8b58d236b04b47f4b833cf6699f6d578 2024-11-13T05:53:15,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6720a2d9acb64573b8bf7b7e4a604903 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6720a2d9acb64573b8bf7b7e4a604903 2024-11-13T05:53:15,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/dbe59616ec624c3cb04626b163f32e51 2024-11-13T05:53:15,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/1d66926fede6494aba799d8277eee431 2024-11-13T05:53:15,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6ddc7e9ecc8a46799ba92f7ea676ba01 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/6ddc7e9ecc8a46799ba92f7ea676ba01 2024-11-13T05:53:15,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/d8fbac8e73514e0ca48307faa9adad3a 2024-11-13T05:53:15,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/399ff72e3d0c45c1bcff19f38b01fd84 2024-11-13T05:53:15,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/93c9feac17ad4158b399aec8d62d9354 2024-11-13T05:53:15,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/fd24063feae646bdb692994e85272993, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5671c70bff8346d8a8e4330257cc819d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/11f0fb29b3ca4ce8bdc73d7d05f88a87, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec69044376d14e38b5f7e5af4b31a0d6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e56dd0a2a86045f0b3f28b0bbc1ef33d, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/2e54af0e23894c4b989e6618322acb4b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5cd76bfdc72e4d4d899085d65c7251eb, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/aa581a094fc44faeb62ad496d7562747, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dc05375a643748a8bf829d1d0a450a7a, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/f559d2e24f8e486785b0fcd6e5a01c8f, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec939430f34e4d4baad3765368f58ff1, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/362111fe7de44a3da4b77cf0e32fa2b8, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/58ecd192b940424daa6d6cfbf8e4f78b, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e78cf2ed61794a11845b51552d75d636, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/b5aaf6760a024ab48a1aabeaf4bc2859, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/8b28bd1d233542828ce7ec6b0d1de54c, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a4f21c5f2b0942fe94839f00e3037b33, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dff1233adc484ace8913a6ce258b24f5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/84be606fb09b48f0b3c3a4475b47267e, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a44e30dce7ef487594ec5c2505e32817, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0dc81dfdf7e24de2ae580bdd99d8b383] to archive 2024-11-13T05:53:15,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:53:15,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/fd24063feae646bdb692994e85272993 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/fd24063feae646bdb692994e85272993 2024-11-13T05:53:15,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5671c70bff8346d8a8e4330257cc819d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5671c70bff8346d8a8e4330257cc819d 2024-11-13T05:53:15,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/11f0fb29b3ca4ce8bdc73d7d05f88a87 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/11f0fb29b3ca4ce8bdc73d7d05f88a87 2024-11-13T05:53:15,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec69044376d14e38b5f7e5af4b31a0d6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec69044376d14e38b5f7e5af4b31a0d6 2024-11-13T05:53:15,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e56dd0a2a86045f0b3f28b0bbc1ef33d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e56dd0a2a86045f0b3f28b0bbc1ef33d 2024-11-13T05:53:15,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/2e54af0e23894c4b989e6618322acb4b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/2e54af0e23894c4b989e6618322acb4b 2024-11-13T05:53:15,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5cd76bfdc72e4d4d899085d65c7251eb to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/5cd76bfdc72e4d4d899085d65c7251eb 2024-11-13T05:53:15,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/aa581a094fc44faeb62ad496d7562747 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/aa581a094fc44faeb62ad496d7562747 2024-11-13T05:53:15,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dc05375a643748a8bf829d1d0a450a7a to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dc05375a643748a8bf829d1d0a450a7a 2024-11-13T05:53:15,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/f559d2e24f8e486785b0fcd6e5a01c8f to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/f559d2e24f8e486785b0fcd6e5a01c8f 2024-11-13T05:53:15,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec939430f34e4d4baad3765368f58ff1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/ec939430f34e4d4baad3765368f58ff1 2024-11-13T05:53:15,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/362111fe7de44a3da4b77cf0e32fa2b8 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/362111fe7de44a3da4b77cf0e32fa2b8 2024-11-13T05:53:15,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/58ecd192b940424daa6d6cfbf8e4f78b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/58ecd192b940424daa6d6cfbf8e4f78b 2024-11-13T05:53:15,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e78cf2ed61794a11845b51552d75d636 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/e78cf2ed61794a11845b51552d75d636 2024-11-13T05:53:15,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/b5aaf6760a024ab48a1aabeaf4bc2859 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/b5aaf6760a024ab48a1aabeaf4bc2859 2024-11-13T05:53:15,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/8b28bd1d233542828ce7ec6b0d1de54c to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/8b28bd1d233542828ce7ec6b0d1de54c 2024-11-13T05:53:15,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a4f21c5f2b0942fe94839f00e3037b33 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a4f21c5f2b0942fe94839f00e3037b33 2024-11-13T05:53:15,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dff1233adc484ace8913a6ce258b24f5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/dff1233adc484ace8913a6ce258b24f5 2024-11-13T05:53:15,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/84be606fb09b48f0b3c3a4475b47267e to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/84be606fb09b48f0b3c3a4475b47267e 2024-11-13T05:53:15,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a44e30dce7ef487594ec5c2505e32817 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/a44e30dce7ef487594ec5c2505e32817 2024-11-13T05:53:15,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0dc81dfdf7e24de2ae580bdd99d8b383 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0dc81dfdf7e24de2ae580bdd99d8b383 2024-11-13T05:53:15,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/4ebaf5b263ce4492abbc6b8dfe5e9532, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/5ef7b7b3b664427ba04e02552c0c61df, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/074da16739ea4c46a12fdf9c98ab0184, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/83d8ced5556a493b8bae57d622d30e14, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/336dc73504c34e338e4d43897cedd235, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/de3fb15113be474f8a5011e60f3bcffe, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9d9f4174719849e883383aa034a31705, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/a05f8aa1e607465f959f788744920a20, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/595e335e6a4a4e46b75e1a792ff231da, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/27d5fd6121fe46d8b352aac7b2785e20, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/f09f5f7406354044a90a6e76e39cbe05, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/2b26667d94794d419003332138f8c428, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/1011a363e7374f139194bd8dd3c65abd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9ee401da56d649a4bdc495069a3d1fcd, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/74ae82e181c54e8b95f0214c8983b7c5, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/7bd78d23021245a3a8e1b32ede7ebd42, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/447ff5ced68c458e949ebacdbaaa4997, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/db93a2e3ecad41269b1fa7607c1d81ef, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/bc274652c0ac4b0aa48c127ce6bd2869, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/126dfd195fab43f7878c2db6eb4ab7e6, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/ee93c66bb5314d7a868e940a895f0842] to archive 2024-11-13T05:53:15,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:53:15,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/4ebaf5b263ce4492abbc6b8dfe5e9532 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/4ebaf5b263ce4492abbc6b8dfe5e9532 2024-11-13T05:53:15,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/5ef7b7b3b664427ba04e02552c0c61df to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/5ef7b7b3b664427ba04e02552c0c61df 2024-11-13T05:53:15,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/074da16739ea4c46a12fdf9c98ab0184 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/074da16739ea4c46a12fdf9c98ab0184 2024-11-13T05:53:15,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/83d8ced5556a493b8bae57d622d30e14 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/83d8ced5556a493b8bae57d622d30e14 2024-11-13T05:53:15,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/336dc73504c34e338e4d43897cedd235 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/336dc73504c34e338e4d43897cedd235 2024-11-13T05:53:15,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/de3fb15113be474f8a5011e60f3bcffe to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/de3fb15113be474f8a5011e60f3bcffe 2024-11-13T05:53:15,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9d9f4174719849e883383aa034a31705 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9d9f4174719849e883383aa034a31705 2024-11-13T05:53:15,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/a05f8aa1e607465f959f788744920a20 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/a05f8aa1e607465f959f788744920a20 2024-11-13T05:53:15,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/595e335e6a4a4e46b75e1a792ff231da to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/595e335e6a4a4e46b75e1a792ff231da 2024-11-13T05:53:15,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/27d5fd6121fe46d8b352aac7b2785e20 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/27d5fd6121fe46d8b352aac7b2785e20 2024-11-13T05:53:15,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/f09f5f7406354044a90a6e76e39cbe05 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/f09f5f7406354044a90a6e76e39cbe05 2024-11-13T05:53:15,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/2b26667d94794d419003332138f8c428 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/2b26667d94794d419003332138f8c428 2024-11-13T05:53:15,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/1011a363e7374f139194bd8dd3c65abd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/1011a363e7374f139194bd8dd3c65abd 2024-11-13T05:53:15,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9ee401da56d649a4bdc495069a3d1fcd to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/9ee401da56d649a4bdc495069a3d1fcd 2024-11-13T05:53:15,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/74ae82e181c54e8b95f0214c8983b7c5 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/74ae82e181c54e8b95f0214c8983b7c5 2024-11-13T05:53:15,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/7bd78d23021245a3a8e1b32ede7ebd42 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/7bd78d23021245a3a8e1b32ede7ebd42 2024-11-13T05:53:15,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/447ff5ced68c458e949ebacdbaaa4997 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/447ff5ced68c458e949ebacdbaaa4997 2024-11-13T05:53:15,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/db93a2e3ecad41269b1fa7607c1d81ef to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/db93a2e3ecad41269b1fa7607c1d81ef 2024-11-13T05:53:15,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/bc274652c0ac4b0aa48c127ce6bd2869 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/bc274652c0ac4b0aa48c127ce6bd2869 2024-11-13T05:53:15,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/126dfd195fab43f7878c2db6eb4ab7e6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/126dfd195fab43f7878c2db6eb4ab7e6 2024-11-13T05:53:15,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/ee93c66bb5314d7a868e940a895f0842 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/ee93c66bb5314d7a868e940a895f0842 2024-11-13T05:53:15,070 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=4 2024-11-13T05:53:15,070 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. 2024-11-13T05:53:15,070 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 636fe54399d3e25dd08446a6e1c82434: Waiting for close lock at 1731477194150Running coprocessor pre-close hooks at 1731477194150Disabling compacts and flushes for region at 1731477194150Disabling writes for close at 1731477194150Obtaining lock to block concurrent updates at 1731477194150Preparing flush snapshotting stores in 636fe54399d3e25dd08446a6e1c82434 at 1731477194150Finished memstore snapshotting TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731477194150Flushing stores of TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434. at 1731477194151 (+1 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/A: creating writer at 1731477194151Flushing mob file 636fe54399d3e25dd08446a6e1c82434/A: appending metadata at 1731477194156 (+5 ms)Flushing mob file 636fe54399d3e25dd08446a6e1c82434/A: closing flushed file at 1731477194156Flushing 636fe54399d3e25dd08446a6e1c82434/A: appending metadata at 1731477194167 (+11 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/A: closing flushed file at 1731477194168 (+1 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/B: creating writer at 1731477194176 (+8 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/B: appending metadata at 1731477194179 (+3 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/B: closing flushed file at 1731477194179Flushing 636fe54399d3e25dd08446a6e1c82434/C: creating writer at 1731477194587 (+408 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/C: appending metadata at 1731477194590 (+3 ms)Flushing 636fe54399d3e25dd08446a6e1c82434/C: closing flushed file at 1731477194590Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b2d1169: reopening flushed file at 1731477195002 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1eea0c: reopening flushed file at 1731477195006 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6835415d: reopening flushed file at 1731477195009 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 636fe54399d3e25dd08446a6e1c82434 in 863ms, sequenceid=350, compaction requested=true at 1731477195013 (+4 ms)Writing region close event to WAL at 1731477195068 (+55 ms)Running coprocessor post-close hooks at 1731477195070 (+2 ms)Closed at 1731477195070 2024-11-13T05:53:15,072 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=636fe54399d3e25dd08446a6e1c82434, regionState=CLOSED 2024-11-13T05:53:15,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 because future has completed 2024-11-13T05:53:15,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-13T05:53:15,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 636fe54399d3e25dd08446a6e1c82434, server=bc386b7bf2fe,38979,1731477016232 in 1.0770 sec 2024-11-13T05:53:15,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-13T05:53:15,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=636fe54399d3e25dd08446a6e1c82434, UNASSIGN in 1.0820 sec 2024-11-13T05:53:15,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T05:53:15,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0860 sec 2024-11-13T05:53:15,080 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477195080"}]},"ts":"1731477195080"} 2024-11-13T05:53:15,081 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T05:53:15,081 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T05:53:15,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0960 sec 2024-11-13T05:53:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T05:53:15,121 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:53:15,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T05:53:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:15,123 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T05:53:15,124 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:15,125 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,127 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C, FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/recovered.edits] 2024-11-13T05:53:15,129 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4424c6e8320f4333ace5dba2f1233df1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/4424c6e8320f4333ace5dba2f1233df1 2024-11-13T05:53:15,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/775f052707104ce39a230156de2b24f1 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/775f052707104ce39a230156de2b24f1 2024-11-13T05:53:15,131 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7f3b1afe86fb40739f829916dfd9c023 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/A/7f3b1afe86fb40739f829916dfd9c023 2024-11-13T05:53:15,132 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0afc710896a14338ade0d4a18b6dd040 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/0afc710896a14338ade0d4a18b6dd040 2024-11-13T05:53:15,133 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/554544fd5da74990a30982c001a62e30 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/554544fd5da74990a30982c001a62e30 2024-11-13T05:53:15,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/579b1f60db124826a48617653ff8faa6 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/B/579b1f60db124826a48617653ff8faa6 2024-11-13T05:53:15,135 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/0a31888562214e699de54e24d3f3301b to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/0a31888562214e699de54e24d3f3301b 2024-11-13T05:53:15,136 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/469982ea945645baaeaff3bb6d9ba70d to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/469982ea945645baaeaff3bb6d9ba70d 2024-11-13T05:53:15,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/e797384c3ab6471a90bfebd5822f26b4 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/C/e797384c3ab6471a90bfebd5822f26b4 2024-11-13T05:53:15,138 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/recovered.edits/353.seqid to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434/recovered.edits/353.seqid 2024-11-13T05:53:15,139 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/default/TestAcidGuarantees/636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,139 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T05:53:15,139 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:53:15,140 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T05:53:15,142 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111300e474c52ca44b59a58202c45e19fc52_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111300e474c52ca44b59a58202c45e19fc52_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,143 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304e2c07763134eba814b61a556c00216_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304e2c07763134eba814b61a556c00216_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,144 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e3aaa85ba6d45dc8fb25bf986ee59e2_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e3aaa85ba6d45dc8fb25bf986ee59e2_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,144 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f64153a48bd467cab475a3df291733b_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411132f64153a48bd467cab475a3df291733b_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,145 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113524351604b86498cb69fbf72c84bf8db_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113524351604b86498cb69fbf72c84bf8db_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,146 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366ec59a01fa14f488be3a7dc5ce255f4_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111366ec59a01fa14f488be3a7dc5ce255f4_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,147 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f044bda47ee44f3a8a11741c35bdc20_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f044bda47ee44f3a8a11741c35bdc20_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,147 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f2b403317a84cb7ba8aecadfe6e67cf_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f2b403317a84cb7ba8aecadfe6e67cf_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,148 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111374569f2a61ac4d21af6f68c7cd085492_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111374569f2a61ac4d21af6f68c7cd085492_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,149 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137c9ae97da7c74bbe8dbea82d6430a074_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411137c9ae97da7c74bbe8dbea82d6430a074_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,149 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113826c1084c7fd40528c3561c949640139_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113826c1084c7fd40528c3561c949640139_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,150 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383cd3dc9a8d5474faaf74a84dad93478_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111383cd3dc9a8d5474faaf74a84dad93478_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,151 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113955765d24cba45ee82571b811fcbb37a_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113955765d24cba45ee82571b811fcbb37a_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,151 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b0141e9c882b4eaa8efb6945359f9460_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b0141e9c882b4eaa8efb6945359f9460_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,152 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bdbc57c6f6f24554afc4836b55f067e8_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bdbc57c6f6f24554afc4836b55f067e8_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,153 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d6191af4f3fd4e0d82193b92b6b4ca87_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d6191af4f3fd4e0d82193b92b6b4ca87_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,153 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8fcf3b3d40c4f91874e7767db2a6685_636fe54399d3e25dd08446a6e1c82434 to hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f8fcf3b3d40c4f91874e7767db2a6685_636fe54399d3e25dd08446a6e1c82434 2024-11-13T05:53:15,154 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T05:53:15,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:15,157 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T05:53:15,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T05:53:15,160 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:15,160 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T05:53:15,160 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731477195160"}]},"ts":"9223372036854775807"} 2024-11-13T05:53:15,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T05:53:15,162 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 636fe54399d3e25dd08446a6e1c82434, NAME => 'TestAcidGuarantees,,1731477162523.636fe54399d3e25dd08446a6e1c82434.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T05:53:15,162 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T05:53:15,162 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731477195162"}]},"ts":"9223372036854775807"} 2024-11-13T05:53:15,163 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T05:53:15,164 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T05:53:15,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-13T05:53:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44267 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T05:53:15,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T05:53:15,231 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T05:53:15,239 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1501 (was 350) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7559 (was 7619) 2024-11-13T05:53:15,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:53:15,240 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:53:15,240 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:53:15,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:15,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:15,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:53:15,240 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:53:15,240 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-13T05:53:15,241 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bc386b7bf2fe,44267,1731477015408 2024-11-13T05:53:15,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:53:15,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:53:15,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:53:15,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:53:15,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:53:15,277 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:53:15,277 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:53:15,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:15,277 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,38979,1731477016232' ***** 2024-11-13T05:53:15,278 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:53:15,278 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:53:15,278 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:53:15,278 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:53:15,278 INFO [RS:0;bc386b7bf2fe:38979 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:53:15,278 INFO [RS:0;bc386b7bf2fe:38979 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:53:15,279 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bc386b7bf2fe:38979. 2024-11-13T05:53:15,279 DEBUG [RS:0;bc386b7bf2fe:38979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:53:15,279 DEBUG [RS:0;bc386b7bf2fe:38979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:53:15,279 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:53:15,280 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T05:53:15,280 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:53:15,281 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:53:15,281 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:15,281 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:53:15,281 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:53:15,281 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:53:15,281 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:53:15,281 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T05:53:15,305 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/info/1b78f769b59b46aba4fc4f59cd73d79b is 95, key is TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91./info:/1731477048021/DeleteFamily/seqid=0 2024-11-13T05:53:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742482_1658 (size=6514) 2024-11-13T05:53:15,312 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/info/1b78f769b59b46aba4fc4f59cd73d79b 2024-11-13T05:53:15,330 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/ns/561fc39f70ce410eb45c179f19e3e7a8 is 93, key is TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91./ns:/1731477047985/DeleteFamily/seqid=0 2024-11-13T05:53:15,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742483_1659 (size=5906) 2024-11-13T05:53:15,346 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:53:15,481 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:15,681 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:15,728 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:53:15,728 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:53:15,734 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/ns/561fc39f70ce410eb45c179f19e3e7a8 2024-11-13T05:53:15,752 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/rep_barrier/ea31f17bfd014666ade062671bd2a6e5 is 102, key is TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91./rep_barrier:/1731477047985/DeleteFamily/seqid=0 2024-11-13T05:53:15,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742484_1660 (size=6025) 2024-11-13T05:53:15,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T05:53:15,881 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:16,081 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:16,155 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/rep_barrier/ea31f17bfd014666ade062671bd2a6e5 2024-11-13T05:53:16,176 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/table/f8a003ce33de48a885997d2d7583ac3f is 96, key is TestAcidGuarantees,,1731477021349.9480a359b149cb565364fd0f1f82db91./table:/1731477047985/DeleteFamily/seqid=0 2024-11-13T05:53:16,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742485_1661 (size=5936) 2024-11-13T05:53:16,282 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:16,482 DEBUG [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:53:16,580 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/table/f8a003ce33de48a885997d2d7583ac3f 2024-11-13T05:53:16,583 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/info/1b78f769b59b46aba4fc4f59cd73d79b as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/info/1b78f769b59b46aba4fc4f59cd73d79b 2024-11-13T05:53:16,586 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/info/1b78f769b59b46aba4fc4f59cd73d79b, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T05:53:16,587 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/ns/561fc39f70ce410eb45c179f19e3e7a8 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/ns/561fc39f70ce410eb45c179f19e3e7a8 2024-11-13T05:53:16,589 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/ns/561fc39f70ce410eb45c179f19e3e7a8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T05:53:16,590 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/rep_barrier/ea31f17bfd014666ade062671bd2a6e5 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/rep_barrier/ea31f17bfd014666ade062671bd2a6e5 2024-11-13T05:53:16,593 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/rep_barrier/ea31f17bfd014666ade062671bd2a6e5, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T05:53:16,593 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/.tmp/table/f8a003ce33de48a885997d2d7583ac3f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/table/f8a003ce33de48a885997d2d7583ac3f 2024-11-13T05:53:16,596 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/table/f8a003ce33de48a885997d2d7583ac3f, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T05:53:16,596 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1315ms, sequenceid=90, compaction requested=false 2024-11-13T05:53:16,601 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T05:53:16,601 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:53:16,601 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:53:16,601 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477195280Running coprocessor pre-close hooks at 1731477195281 (+1 ms)Disabling compacts and flushes for region at 1731477195281Disabling writes for close at 1731477195281Obtaining lock to block concurrent updates at 1731477195281Preparing flush snapshotting stores in 1588230740 at 1731477195281Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731477195281Flushing stores of hbase:meta,,1.1588230740 at 1731477195282 (+1 ms)Flushing 1588230740/info: creating writer at 1731477195282Flushing 1588230740/info: appending metadata at 1731477195305 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731477195305Flushing 1588230740/ns: creating writer at 1731477195315 (+10 ms)Flushing 1588230740/ns: appending metadata at 1731477195330 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731477195330Flushing 1588230740/rep_barrier: creating writer at 1731477195738 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731477195751 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731477195751Flushing 1588230740/table: creating writer at 1731477196159 (+408 ms)Flushing 1588230740/table: appending metadata at 1731477196176 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731477196176Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36768b9c: reopening flushed file at 1731477196583 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22d09a06: reopening flushed file at 1731477196586 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c687634: reopening flushed file at 1731477196590 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@504cb580: reopening flushed file at 1731477196593 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1315ms, sequenceid=90, compaction requested=false at 1731477196596 (+3 ms)Writing region close event to WAL at 1731477196598 (+2 ms)Running coprocessor post-close hooks at 1731477196601 (+3 ms)Closed at 1731477196601 2024-11-13T05:53:16,601 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:53:16,682 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,38979,1731477016232; all regions closed. 2024-11-13T05:53:16,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741834_1010 (size=25344) 2024-11-13T05:53:16,689 DEBUG [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/oldWALs 2024-11-13T05:53:16,689 INFO [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C38979%2C1731477016232.meta:.meta(num 1731477020832) 2024-11-13T05:53:16,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741832_1008 (size=16573799) 2024-11-13T05:53:16,692 DEBUG [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/oldWALs 2024-11-13T05:53:16,692 INFO [RS:0;bc386b7bf2fe:38979 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C38979%2C1731477016232:(num 1731477019881) 2024-11-13T05:53:16,692 DEBUG [RS:0;bc386b7bf2fe:38979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:53:16,692 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:53:16,692 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:53:16,693 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:53:16,693 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:53:16,693 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:53:16,694 INFO [RS:0;bc386b7bf2fe:38979 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38979 2024-11-13T05:53:16,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:53:16,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,38979,1731477016232 2024-11-13T05:53:16,726 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:53:16,735 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,38979,1731477016232] 2024-11-13T05:53:16,743 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,38979,1731477016232 already deleted, retry=false 2024-11-13T05:53:16,743 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,38979,1731477016232 expired; onlineServers=0 2024-11-13T05:53:16,743 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bc386b7bf2fe,44267,1731477015408' ***** 2024-11-13T05:53:16,743 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:53:16,743 INFO [M:0;bc386b7bf2fe:44267 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:53:16,743 INFO [M:0;bc386b7bf2fe:44267 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:53:16,743 DEBUG [M:0;bc386b7bf2fe:44267 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:53:16,744 DEBUG [M:0;bc386b7bf2fe:44267 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:53:16,744 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477019553 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477019553,5,FailOnTimeoutGroup] 2024-11-13T05:53:16,744 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477019554 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477019554,5,FailOnTimeoutGroup] 2024-11-13T05:53:16,744 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:53:16,744 INFO [M:0;bc386b7bf2fe:44267 {}] hbase.ChoreService(370): Chore service for: master/bc386b7bf2fe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:53:16,744 INFO [M:0;bc386b7bf2fe:44267 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:53:16,744 DEBUG [M:0;bc386b7bf2fe:44267 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:53:16,744 INFO [M:0;bc386b7bf2fe:44267 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:53:16,744 INFO [M:0;bc386b7bf2fe:44267 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:53:16,744 ERROR [M:0;bc386b7bf2fe:44267 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T05:53:16,745 INFO [M:0;bc386b7bf2fe:44267 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:53:16,745 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:53:16,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:53:16,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:53:16,752 DEBUG [M:0;bc386b7bf2fe:44267 {}] zookeeper.ZKUtil(347): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:53:16,752 WARN [M:0;bc386b7bf2fe:44267 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:53:16,752 INFO [M:0;bc386b7bf2fe:44267 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/.lastflushedseqids 2024-11-13T05:53:16,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742486_1662 (size=72) 2024-11-13T05:53:16,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:53:16,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x10134081ae10001, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:53:16,836 INFO [RS:0;bc386b7bf2fe:38979 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:53:16,836 INFO [RS:0;bc386b7bf2fe:38979 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,38979,1731477016232; zookeeper connection closed. 2024-11-13T05:53:16,836 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d4f0802 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d4f0802 2024-11-13T05:53:16,837 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T05:53:17,162 INFO [M:0;bc386b7bf2fe:44267 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:53:17,163 INFO [M:0;bc386b7bf2fe:44267 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:53:17,163 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:53:17,163 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:53:17,163 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:53:17,163 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:53:17,163 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:53:17,163 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=797.88 KB heapSize=979.05 KB 2024-11-13T05:53:17,177 DEBUG [M:0;bc386b7bf2fe:44267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee0a9eab8d7140788884a00698d27206 is 82, key is hbase:meta,,1/info:regioninfo/1731477020911/Put/seqid=0 2024-11-13T05:53:17,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742487_1663 (size=5672) 2024-11-13T05:53:17,580 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2248 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee0a9eab8d7140788884a00698d27206 2024-11-13T05:53:17,601 DEBUG [M:0;bc386b7bf2fe:44267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb472b87b72644b2ac10f1640dbf3b7f is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1731477165713/Put/seqid=0 2024-11-13T05:53:17,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742488_1664 (size=45078) 2024-11-13T05:53:18,005 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=797.33 KB at sequenceid=2248 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb472b87b72644b2ac10f1640dbf3b7f 2024-11-13T05:53:18,012 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb472b87b72644b2ac10f1640dbf3b7f 2024-11-13T05:53:18,032 DEBUG [M:0;bc386b7bf2fe:44267 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/87ba593ff41240e0a5458e129e6d954c is 69, key is bc386b7bf2fe,38979,1731477016232/rs:state/1731477019611/Put/seqid=0 2024-11-13T05:53:18,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073742489_1665 (size=5156) 2024-11-13T05:53:18,435 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2248 (bloomFilter=true), to=hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/87ba593ff41240e0a5458e129e6d954c 2024-11-13T05:53:18,439 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ee0a9eab8d7140788884a00698d27206 as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ee0a9eab8d7140788884a00698d27206 2024-11-13T05:53:18,442 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ee0a9eab8d7140788884a00698d27206, entries=8, sequenceid=2248, filesize=5.5 K 2024-11-13T05:53:18,443 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb472b87b72644b2ac10f1640dbf3b7f as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb472b87b72644b2ac10f1640dbf3b7f 2024-11-13T05:53:18,446 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb472b87b72644b2ac10f1640dbf3b7f 2024-11-13T05:53:18,446 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb472b87b72644b2ac10f1640dbf3b7f, entries=168, sequenceid=2248, filesize=44.0 K 2024-11-13T05:53:18,447 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/87ba593ff41240e0a5458e129e6d954c as hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/87ba593ff41240e0a5458e129e6d954c 2024-11-13T05:53:18,450 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41183/user/jenkins/test-data/ac8e7679-8e71-4308-2d3c-ff9a49474468/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/87ba593ff41240e0a5458e129e6d954c, entries=1, sequenceid=2248, filesize=5.0 K 2024-11-13T05:53:18,451 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(3140): Finished flush of dataSize ~797.88 KB/817034, heapSize ~978.75 KB/1002240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=2248, compaction requested=false 2024-11-13T05:53:18,452 INFO [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:53:18,452 DEBUG [M:0;bc386b7bf2fe:44267 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477197163Disabling compacts and flushes for region at 1731477197163Disabling writes for close at 1731477197163Obtaining lock to block concurrent updates at 1731477197163Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731477197163Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=817034, getHeapSize=1002480, getOffHeapSize=0, getCellsCount=2441 at 1731477197163Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731477197164 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731477197164Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731477197177 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731477197177Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731477197583 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731477197601 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731477197601Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731477198012 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731477198031 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731477198031Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62cb5c4d: reopening flushed file at 1731477198438 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ab5fc3a: reopening flushed file at 1731477198442 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c09378: reopening flushed file at 1731477198446 (+4 ms)Finished flush of dataSize ~797.88 KB/817034, heapSize ~978.75 KB/1002240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1288ms, sequenceid=2248, compaction requested=false at 1731477198451 (+5 ms)Writing region close event to WAL at 1731477198452 (+1 ms)Closed at 1731477198452 2024-11-13T05:53:18,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34273 is added to blk_1073741830_1006 (size=964345) 2024-11-13T05:53:18,454 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:53:18,454 INFO [M:0;bc386b7bf2fe:44267 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:53:18,454 INFO [M:0;bc386b7bf2fe:44267 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44267 2024-11-13T05:53:18,455 INFO [M:0;bc386b7bf2fe:44267 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:53:18,584 INFO [M:0;bc386b7bf2fe:44267 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:53:18,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:53:18,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44267-0x10134081ae10000, quorum=127.0.0.1:49495, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:53:18,589 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:53:18,592 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:53:18,592 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:53:18,592 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:53:18,592 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/hadoop.log.dir/,STOPPED} 2024-11-13T05:53:18,595 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:53:18,595 WARN [BP-959269583-172.17.0.2-1731477011648 heartbeating to localhost/127.0.0.1:41183 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:53:18,595 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:53:18,595 WARN [BP-959269583-172.17.0.2-1731477011648 heartbeating to localhost/127.0.0.1:41183 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959269583-172.17.0.2-1731477011648 (Datanode Uuid a6f45347-bf4f-4995-8b21-bcffbf3db50e) service to localhost/127.0.0.1:41183 2024-11-13T05:53:18,599 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/data/data1/current/BP-959269583-172.17.0.2-1731477011648 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:53:18,599 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/cluster_ae127bdb-d176-baea-ea0c-3a94fc4f7259/data/data2/current/BP-959269583-172.17.0.2-1731477011648 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:53:18,600 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:53:18,609 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:53:18,610 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:53:18,610 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:53:18,610 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:53:18,610 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/23219b38-0011-b416-91d9-7ff461b54a3d/hadoop.log.dir/,STOPPED} 2024-11-13T05:53:18,626 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:53:18,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down